From 10a6c5abbae1730e803b50e700dd46efb935748f Mon Sep 17 00:00:00 2001 From: zhikunyao Date: Wed, 29 Apr 2026 17:31:51 +0800 Subject: [PATCH] doc: add scripts/milvus-debug.sh for coredump analysis with debug image (#49084) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Add `scripts/milvus-debug.sh` to automate GDB coredump analysis using the `-debug` image that accompanies each Milvus release starting from v2.6.14. ## Background Starting with v2.6.14, each Milvus release publishes two image variants: | Tag | Content | Size | Use | |-----|---------|------|-----| | `milvusdb/milvus:` | **stripped** (production) | ~1/3 | Production deployment | | `milvusdb/milvus:-debug` | unstripped (full symbols) | full | GDB coredump analysis | The stripped production image is **byte-identical** to the debug image in its code sections (`strip --strip-debug` removes only `.debug_*` sections and symbol tables). So GDB can use the debug image's symbols to resolve addresses in a coredump produced by the stripped image. ## What the script does 1. Pulls the debug image 2. Mounts the coredump file read-only into a container 3. Installs `gdb` on demand (not present in the runtime image by default) 4. Launches GDB with `solib-search-path=/milvus/lib` so symbols from all shared libraries (`libknowhere.so`, `libtantivy-binding.so`, etc.) resolve correctly 5. Auto-prints the initial backtrace ## Typical usage after a production crash ```bash # 1. Copy the coredump out of the crashed pod kubectl cp /:/tmp/cores/core. ./core. # 2. Run this script (GDB starts automatically with backtrace) ./scripts/milvus-debug.sh ./core. \ milvusdb/milvus:v2.6.14 milvusdb/milvus:v2.6.14-debug ``` ## Test plan - [x] `bash -n scripts/milvus-debug.sh` (syntax check passes) - [x] Mirrors the documented procedure in the strip-image-coredump-debug-guide Feishu doc - [ ] Manual validation on a real coredump (waiting for v2.6.14 production crash; can be exercised with any coredump for now) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Signed-off-by: Zhikun Yao Co-authored-by: Claude Opus 4.6 --- scripts/milvus-debug.sh | 103 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 scripts/milvus-debug.sh diff --git a/scripts/milvus-debug.sh b/scripts/milvus-debug.sh new file mode 100755 index 0000000000..29c7d85e84 --- /dev/null +++ b/scripts/milvus-debug.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Debug a coredump produced by the stripped (production) Milvus image +# using the matching unstripped debug image. +# +# Starting from v2.6.15, each Milvus release publishes two image variants: +# milvusdb/milvus: - stripped, production (default, ~1/3 size) +# milvusdb/milvus:-debug - unstripped, full debug symbols (for GDB) +# +# The stripped and debug images share byte-identical code sections, so GDB +# can use the debug image's symbols to resolve addresses in a coredump +# produced by the stripped image. +# +# Usage: +# ./milvus-debug.sh +# +# Examples: +# ./milvus-debug.sh ./core.12345 milvusdb/milvus:v2.6.15 milvusdb/milvus:v2.6.15-debug +# ./milvus-debug.sh ./core.12345 harbor.example.com/milvus:v2.6.15 harbor.example.com/milvus:v2.6.15-debug +# +# Typical workflow when a Milvus pod crashes in production: +# # 1. Copy the coredump out of the crashed pod +# kubectl cp /:/tmp/cores/core. ./core. +# +# # 2. Run this script (GDB starts automatically with backtrace) +# ./milvus-debug.sh ./core. milvusdb/milvus:v2.6.15 milvusdb/milvus:v2.6.15-debug +# +set -euo pipefail + +COREDUMP="${1:?Usage: $0 }" +IMAGE="${2:?Missing stripped image, e.g. milvusdb/milvus:v2.6.15}" +IMAGE_NON_STRIP="${3:?Missing debug image, e.g. milvusdb/milvus:v2.6.15-debug}" + +# Resolve to absolute path +COREDUMP="$(cd "$(dirname "$COREDUMP")" && pwd)/$(basename "$COREDUMP")" + +if [ ! -f "$COREDUMP" ]; then + echo "ERROR: coredump file not found: $COREDUMP" + exit 1 +fi + +# Warn if stripped and debug images don't share the same base tag +# Expected: : and :-debug +STRIPPED_TAG="${IMAGE##*:}" +DEBUG_TAG="${IMAGE_NON_STRIP##*:}" +if [ "${DEBUG_TAG}" != "${STRIPPED_TAG}-debug" ]; then + echo "WARNING: debug image tag '${DEBUG_TAG}' does not match '${STRIPPED_TAG}-debug'." + echo " Symbols may not align with the coredump addresses." + echo "" +fi + +echo "==> Pulling debug image: ${IMAGE_NON_STRIP}" +docker pull "${IMAGE_NON_STRIP}" + +echo "" +echo "==> Launching GDB inside debug container..." +echo "" +echo " Coredump: ${COREDUMP}" +echo " Stripped image: ${IMAGE}" +echo " Debug image: ${IMAGE_NON_STRIP}" +echo "" +echo "--- GDB will start. Useful commands: ---" +echo " bt - backtrace of current thread" +echo " thread apply all bt - backtrace of all threads" +echo " info threads - list threads" +echo " thread - switch to thread n" +echo " info locals - local variables" +echo " print - inspect variable" +echo " list - show source context" +echo " info sharedlibrary - check .so symbol status" +echo "----------------------------------------" +echo "" + +docker run -it --rm \ + -v "${COREDUMP}:/tmp/core:ro" \ + --entrypoint "" \ + "${IMAGE_NON_STRIP}" \ + bash -c ' + # Locate the milvus binary + MILVUS_BIN=$(find /milvus -name milvus -type f 2>/dev/null | head -1) + if [ -z "$MILVUS_BIN" ]; then + echo "ERROR: milvus binary not found in image" + exit 1 + fi + echo "Using binary: $MILVUS_BIN" + + # Install gdb if not present + if ! command -v gdb &>/dev/null; then + echo "Installing gdb..." + apt-get update -qq && apt-get install -y -qq gdb >/dev/null 2>&1 + fi + + # Set library paths so GDB can resolve all .so symbols + export LD_LIBRARY_PATH="/milvus/lib:${LD_LIBRARY_PATH:-}" + + # Launch GDB + # - solib-search-path: tells GDB where to find unstripped .so files + # - auto-loads symbols for milvus binary + all shared libraries + gdb "$MILVUS_BIN" /tmp/core \ + -ex "set solib-search-path /milvus/lib" \ + -ex "set print pretty on" \ + -ex "bt" + '