From d9f16220c894b5627138a485d79b8b4734130109 Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Sat, 3 Oct 2026 23:01:00 +0530 Subject: [PATCH 1/2] fix(python): read the target's interpreter in austin memory profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit austin finds the interpreter (and libpython, for a shared build) by the path in /proc//maps and opens that path in its own mount namespace. In the agent's container that path is either missing — "Cannot determine the version of the Python interpreter" for any target whose interpreter the image lacks — or the image's own interpreter. Run austin in a private mount namespace with copies of the target's interpreter files bound at those paths. A bind straight from /proc//root is refused across mount namespaces, hence the copy. Also: - annotate a raw memory profile with no samples as "no memory growth" instead of publishing a header-only artefact; other outputs fail with that reason - reduce austin errors to the sentence that explains them, with the PID and binary - drop TestAustin, which profiled a fixed PID and slept 300s - add a CI job that profiles real Python 3.12/3.14 containers with the built image --- .github/workflows/code-verify.yml | 15 ++ internal/agent/profiler/austin_python_test.go | 20 -- internal/agent/profiler/python_austin.go | 197 +++++++++++++++++- .../profiler/python_austin_output_test.go | 158 ++++++++++++++ test/e2e/python-austin.sh | 101 +++++++++ 5 files changed, 463 insertions(+), 28 deletions(-) delete mode 100644 internal/agent/profiler/austin_python_test.go create mode 100755 test/e2e/python-austin.sh diff --git a/.github/workflows/code-verify.yml b/.github/workflows/code-verify.yml index 4811a0f..b3f6e62 100644 --- a/.github/workflows/code-verify.yml +++ b/.github/workflows/code-verify.yml @@ -61,3 +61,18 @@ jobs: file: ${{ matrix.dockerfile }} platforms: linux/amd64 push: false + + # Unit tests fake the austin command, so they can't catch austin failing to + # read a real interpreter — which every Python memory profile did for any + # target whose interpreter path the profiler image lacked. Profile real + # Python containers with the freshly built image instead. + python-austin-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build the python profiler image + run: docker build -f docker/python/Dockerfile -t application-profiler-python:e2e . + + - name: Profile real Python targets + run: test/e2e/python-austin.sh application-profiler-python:e2e diff --git a/internal/agent/profiler/austin_python_test.go b/internal/agent/profiler/austin_python_test.go deleted file mode 100644 index c480d68..0000000 --- a/internal/agent/profiler/austin_python_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package profiler - -import ( - "testing" - "time" - - "github.com/nudgebee/application-profiler/internal/agent/job" - executil "github.com/nudgebee/application-profiler/internal/agent/util/exec" -) - -func TestAustin(t *testing.T) { - commander := executil.NewCommander() - - austinProfiler := NewAustinPythonProfiler(commander, nil) - - job := &job.ProfilingJob{Duration: 2, PID: "49533", Interval: 10 * time.Second} - austinProfiler.SetUp(job) - austinProfiler.Invoke(job) - time.Sleep(300 * time.Second) -} diff --git a/internal/agent/profiler/python_austin.go b/internal/agent/profiler/python_austin.go index fbc7050..dad7b9f 100644 --- a/internal/agent/profiler/python_austin.go +++ b/internal/agent/profiler/python_austin.go @@ -1,14 +1,19 @@ package profiler import ( + "bufio" "bytes" "context" "fmt" "io" "os" "os/exec" + "path/filepath" + "regexp" "strconv" + "strings" "time" + "unicode" "github.com/agrison/go-commons-lang/stringUtils" "github.com/alitto/pond" @@ -36,16 +41,98 @@ const ( // surfaces as 254. It ends every exposure-limited (-x) run, including the // successful ones, so it can't be treated as a failure on its own. austinInterruptedExitCode = 254 + unshareLocation = "unshare" ) +// austinInTargetFSScript runs austin with the target's interpreter files +// mounted at the paths austin will look for them. +// +// austin finds the interpreter (and libpython, for a shared build) by the +// path it reads from /proc//maps, then opens that path in its OWN mount +// namespace — not under /proc//root. In this container that path is +// either missing (a target on any other Python than ours fails with "Cannot +// determine the version of the Python interpreter") or, worse, our own +// interpreter (a python:3.14 target would be read through our python3.14). +// +// The bind can't come straight from /proc//root: the kernel refuses a +// bind whose source lives in another mount namespace (EINVAL). So each file +// is copied out through /proc//root first and the copy is bound, inside +// a private mount namespace so neither the mounts nor the shadowing of our +// own python leak into the rest of the agent (mojo2austin runs on it). +// +// Positional args: ... . +const austinInTargetFSScript = `set -e +pid=$1; n=$2; shift 2 +i=0 +while [ "$i" -lt "$n" ]; do + p=$1; shift + c="/tmp/austin-target-$pid$p" + mkdir -p "$(dirname "$c")" "$(dirname "$p")" + cp "/proc/$pid/root$p" "$c" + [ -e "$p" ] || touch "$p" + mount --bind "$c" "$p" + i=$((i+1)) +done +exec ` + austinLocation + ` "$@"` + // mojoMagic prefixes austin's binary output format. var mojoMagic = []byte{'M', 'O', 'J', 3} -var austinPythonCommand = func(commander executil.Commander, job *job.ProfilingJob, pid string, fileName string) *exec.Cmd { +// procDir is where the target's /proc entries are read from; a var so tests +// can point it at a fake tree. +var procDir = "/proc" + +var austinPythonCommand = func(commander executil.Commander, job *job.ProfilingJob, pid string, fileName string, targetFiles []string) *exec.Cmd { interval := strconv.Itoa(int(job.Interval.Seconds())) - args := []string{} - args = append(args, "-p", pid, "-o", fileName, "-x", interval, "-m") - return commander.Command(austinLocation, args...) + austinArgs := []string{"-p", pid, "-o", fileName, "-x", interval, "-m"} + if len(targetFiles) == 0 { + return commander.Command(austinLocation, austinArgs...) + } + args := []string{"-m", "--propagation", "private", "sh", "-c", austinInTargetFSScript, "sh", pid, strconv.Itoa(len(targetFiles))} + args = append(args, targetFiles...) + args = append(args, austinArgs...) + return commander.Command(unshareLocation, args...) +} + +// austinTargetFiles returns the files austin opens by the path it finds in +// /proc//maps: the interpreter binary and, for a shared build, +// libpython. Only files reachable through /proc//root are returned, so +// the caller can mount every one of them. +func austinTargetFiles(pid string) ([]string, error) { + exe, err := os.Readlink(filepath.Join(procDir, pid, "exe")) + if err != nil { + return nil, errors.Wrapf(err, "could not resolve the interpreter of PID %s", pid) + } + candidates := []string{strings.TrimSuffix(exe, " (deleted)")} + + maps, err := os.ReadFile(filepath.Join(procDir, pid, "maps")) //nolint:gosec // path built from procDir and a numeric PID + if err != nil { + return nil, errors.Wrapf(err, "could not read the memory map of PID %s", pid) + } + for _, line := range strings.Split(string(maps), "\n") { + fields := strings.Fields(line) + if len(fields) < 6 { + continue + } + path := strings.Join(fields[5:], " ") + if strings.HasPrefix(filepath.Base(path), "libpython") { + candidates = append(candidates, path) + } + } + + var files []string + seen := map[string]bool{} + for _, f := range candidates { + if seen[f] || !filepath.IsAbs(f) { + continue + } + seen[f] = true + if _, err := os.Stat(filepath.Join(procDir, pid, "root", f)); err != nil { + continue + } + files = append(files, f) + } + return files, nil } type AustinPythonProfiler struct { @@ -124,13 +211,24 @@ func (p *austinPythonManager) invoke(job *job.ProfilingJob, pid string) (error, if job.OutputType == api.FlameGraph { fileName = common.GetResultFile(common.TmpDir(), job.Tool, api.Raw, pid, job.Iteration) } - cmd := austinPythonCommand(p.commander, job, pid, fileName) + // Without the target's interpreter files austin can only ever fail, but + // still run it: its own error says more than ours would. + targetFiles, err := austinTargetFiles(pid) + if err != nil { + log.DebugLogLn(err.Error()) + } + binary := "unknown binary" + if len(targetFiles) > 0 { + binary = targetFiles[0] + } + cmd := austinPythonCommand(p.commander, job, pid, fileName, targetFiles) cmd.Stdout = &out cmd.Stderr = &stderr - err := cmd.Run() + err = cmd.Run() if err != nil && !austinStoppedOnSignal(err) { log.ErrorLogLn(out.String()) - return errors.Wrapf(err, "could not launch profiler: %s", stderr.String()), time.Since(start) + return errors.Errorf("could not launch profiler: austin (PID %s, %s): %s (%s)", + pid, binary, austinErrorMessage(stderr.String()), err), time.Since(start) } // austin 4 always writes the binary MOJO format; convert it back to the @@ -143,7 +241,28 @@ func (p *austinPythonManager) invoke(job *job.ProfilingJob, pid string) (error, // attached but read nothing — report that instead of publishing a // zero-byte artefact as a success. if file.IsEmpty(fileName) { - return errors.Errorf("no samples collected (PID: %s): %s", pid, stderr.String()), time.Since(start) + return errors.Errorf("no samples collected: austin (PID %s, %s): %s", + pid, binary, austinErrorMessage(stderr.String())), time.Since(start) + } + + // Memory mode records a sample only when the process's memory grows, so + // a process at steady state legitimately yields a header and nothing + // else. Say so: a header-only artefact reads as a broken profile, and the + // flamegraph path would call it "low cpu load". + samples, err := austinSampleCount(fileName) + if err != nil { + return errors.Wrap(err, "could not read the profile"), time.Since(start) + } + if samples == 0 { + msg := fmt.Sprintf("no memory growth observed during the %s window: austin's memory mode records "+ + "allocations that grow the process's memory, so a process at steady state yields no samples", + job.Interval) + if job.OutputType != api.Raw { + return errors.Errorf("PID %s: %s", pid, msg), time.Since(start) + } + if err := appendLine(fileName, "# "+msg); err != nil { + return errors.Wrap(err, "could not annotate the profile"), time.Since(start) + } } // result file name is composed by the job info and the pid @@ -172,6 +291,68 @@ func austinStoppedOnSignal(err error) bool { return errors.As(err, &exitErr) && exitErr.ExitCode() == austinInterruptedExitCode } +var ( + ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[A-Za-z]`) + // austinBannerEnd is the version line that closes austin's ASCII-art + // banner, e.g. "\__,_|\_,_/__/\__|_|_||_| 4.0.0 [musl-gcc 13.3.0]". + austinBannerEnd = regexp.MustCompile(`\d+\.\d+\.\d+ \[[^\]]*\]`) +) + +// austinErrorMessage reduces austin's stderr to the sentence that explains +// the failure: no ANSI colours, no ASCII-art banner, no emoji, and none of +// the "please report an issue" boilerplate after it. +func austinErrorMessage(stderr string) string { + s := ansiEscape.ReplaceAllString(stderr, "") + if loc := austinBannerEnd.FindStringIndex(s); loc != nil { + s = s[loc[1]:] + } + if i := strings.Index(s, "If you are sure"); i >= 0 { + s = s[:i] + } + s = strings.Join(strings.Fields(s), " ") + s = strings.TrimLeftFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) + if s == "" { + return "no error output" + } + const maxLen = 300 + if len(s) > maxLen { + s = s[:maxLen] + "…" + } + return s +} + +// austinSampleCount counts the sample lines in austin's text output. Every +// sample starts with its process ("P;"); header and metadata lines +// start with "#". +func austinSampleCount(fileName string) (int, error) { + f, err := os.Open(fileName) //nolint:gosec // path built by us from TmpDir + if err != nil { + return 0, err + } + defer func() { _ = f.Close() }() + n := 0 + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "P") { + n++ + } + } + return n, scanner.Err() +} + +func appendLine(fileName, line string) error { + f, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY, 0) //nolint:gosec // path built by us from TmpDir + if err != nil { + return err + } + if _, err := f.WriteString(line + "\n"); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + // convertMojo rewrites fileName in place when it holds austin's binary MOJO // output. A no-op for the text format, so it is safe across austin versions. func (p *austinPythonManager) convertMojo(fileName string) error { diff --git a/internal/agent/profiler/python_austin_output_test.go b/internal/agent/profiler/python_austin_output_test.go index c1596dd..4a9c5f9 100644 --- a/internal/agent/profiler/python_austin_output_test.go +++ b/internal/agent/profiler/python_austin_output_test.go @@ -5,7 +5,11 @@ import ( "os/exec" "path/filepath" "testing" + "time" + "github.com/nudgebee/application-profiler/api" + "github.com/nudgebee/application-profiler/internal/agent/job" + "github.com/nudgebee/application-profiler/internal/agent/profiler/common" executil "github.com/nudgebee/application-profiler/internal/agent/util/exec" "github.com/nudgebee/application-profiler/internal/agent/util/publish" "github.com/stretchr/testify/assert" @@ -79,3 +83,157 @@ func Test_convertMojo(t *testing.T) { require.NoError(t, m.convertMojo(profile)) }) } + +// fakeProc builds a /proc/ tree: an exe link, a maps file, and the +// files present under the target's root. +func fakeProc(t *testing.T, pid, exe, maps string, rootFiles ...string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, pid, "root"), 0o755)) + require.NoError(t, os.Symlink(exe, filepath.Join(dir, pid, "exe"))) + require.NoError(t, os.WriteFile(filepath.Join(dir, pid, "maps"), []byte(maps), 0o600)) + for _, f := range rootFiles { + p := filepath.Join(dir, pid, "root", f) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, nil, 0o600)) + } + return dir +} + +func useProcDir(t *testing.T, dir string) { + t.Helper() + old := procDir + procDir = dir + t.Cleanup(func() { procDir = old }) +} + +// Test_austinTargetFiles — austin opens the interpreter and libpython by the +// paths in /proc//maps, so those are exactly the files to mount. +func Test_austinTargetFiles(t *testing.T) { + const maps = `55d0c0a00000-55d0c0a01000 r--p 00000000 00:2e 1234 /usr/local/bin/python3.12 +7f1c40000000-7f1c40200000 r--p 00000000 00:2e 1235 /usr/local/lib/libpython3.12.so.1.0 +7f1c40200000-7f1c40400000 r-xp 00200000 00:2e 1235 /usr/local/lib/libpython3.12.so.1.0 +7f1c50000000-7f1c50100000 r-xp 00000000 00:2e 1236 /lib/ld-musl-x86_64.so.1 +7f1c60000000-7f1c60100000 r-xp 00000000 00:2e 1237 /usr/local/lib/python3.12/lib-dynload/_json.cpython-312-x86_64-linux-musl.so +7ffd10000000-7ffd10021000 rw-p 00000000 00:00 0 [stack] +` + + t.Run("interpreter first, then libpython, each once", func(t *testing.T) { + useProcDir(t, fakeProc(t, "42", "/usr/local/bin/python3.12", maps, + "/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0", "/lib/ld-musl-x86_64.so.1")) + + files, err := austinTargetFiles("42") + require.NoError(t, err) + assert.Equal(t, []string{"/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0"}, files) + }) + + t.Run("a file missing from the target's root is skipped", func(t *testing.T) { + useProcDir(t, fakeProc(t, "42", "/usr/local/bin/python3.12", maps, "/usr/local/bin/python3.12")) + + files, err := austinTargetFiles("42") + require.NoError(t, err) + assert.Equal(t, []string{"/usr/local/bin/python3.12"}, files) + }) + + t.Run("a replaced interpreter keeps its path", func(t *testing.T) { + useProcDir(t, fakeProc(t, "42", "/usr/bin/python3.11 (deleted)", "", "/usr/bin/python3.11")) + + files, err := austinTargetFiles("42") + require.NoError(t, err) + assert.Equal(t, []string{"/usr/bin/python3.11"}, files) + }) + + t.Run("an unknown PID is an error", func(t *testing.T) { + useProcDir(t, t.TempDir()) + + _, err := austinTargetFiles("42") + assert.Error(t, err) + }) +} + +func Test_austinPythonCommand(t *testing.T) { + j := &job.ProfilingJob{Interval: 30 * time.Second} + + t.Run("no target files runs austin directly", func(t *testing.T) { + cmd := austinPythonCommand(executil.NewCommander(), j, "42", "/tmp/out", nil) + assert.Equal(t, []string{"austin", "-p", "42", "-o", "/tmp/out", "-x", "30", "-m"}, cmd.Args) + }) + + t.Run("target files run austin in a private mount namespace", func(t *testing.T) { + files := []string{"/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0"} + cmd := austinPythonCommand(executil.NewCommander(), j, "42", "/tmp/out", files) + assert.Equal(t, []string{ + "unshare", "-m", "--propagation", "private", "sh", "-c", austinInTargetFSScript, "sh", + "42", "2", "/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0", + "-p", "42", "-o", "/tmp/out", "-x", "30", "-m", + }, cmd.Args) + }) +} + +// Test_austinErrorMessage uses the stderr austin 4.0.0 actually prints when +// it cannot read the interpreter. +func Test_austinErrorMessage(t *testing.T) { + stderr := "\x1b[1m _ _ \x1b[0m\n\x1b[1m __ _ _ _ __| |_(_)_ _ \x1b[0m\n" + + "\x1b[1m/ _` | || (_-< _| | ' \\ \x1b[0m\n\x1b[1m\\__,_|\\_,_/__/\\__|_|_||_|\x1b[0m \x1b[1;36m4.0.0\x1b[0m [musl-gcc 13.3.0]\n\n" + + "🔢 Cannot determine the version of the Python interpreter. This could be due\n" + + "to the binary not being an actual Python binary, like uWSGI, or a version that\n" + + "this version of Austin does not support. If you are sure that the binary is a\n" + + "supported Python binary, please report an issue at\n\n 🌐 https://github.com/P403n1x87/austin/issues\n" + + assert.Equal(t, "Cannot determine the version of the Python interpreter. This could be due to the binary "+ + "not being an actual Python binary, like uWSGI, or a version that this version of Austin does not support.", + austinErrorMessage(stderr)) + assert.Equal(t, "Permission denied", austinErrorMessage("Permission denied\n")) + assert.Equal(t, "no error output", austinErrorMessage("")) +} + +func Test_austinSampleCount(t *testing.T) { + profile := filepath.Join(t.TempDir(), "profile.raw") + require.NoError(t, os.WriteFile(profile, []byte("# austin: 4.0.0\n# mode: memory\n\nP1;T0:1;/app.py::6 131072\nP1;T0:1;/app.py::6 65536\n"), 0o600)) + + n, err := austinSampleCount(profile) + require.NoError(t, err) + assert.Equal(t, 2, n) +} + +// Test_invoke_noMemoryGrowth — memory mode only samples growth, so a steady +// process gives a header and nothing else. A raw profile says so and still +// publishes; any other output fails with the same explanation. +func Test_invoke_noMemoryGrowth(t *testing.T) { + setup := func(t *testing.T, output api.OutputType) (*austinPythonManager, publish.FakePublisher, *job.ProfilingJob, string) { + t.Helper() + tmp := t.TempDir() + oldTmp := common.TmpDir + common.TmpDir = func() string { return tmp } + t.Cleanup(func() { common.TmpDir = oldTmp }) + useProcDir(t, t.TempDir()) + + j := &job.ProfilingJob{Tool: api.Austin, OutputType: output, Interval: 30 * time.Second, Iteration: 1} + raw := common.GetResultFile(tmp, j.Tool, api.Raw, "42", j.Iteration) + commander := executil.NewFakeCommander() + commander.On("Command").Return(exec.Command("sh", "-c", "printf '# austin: 4.0.0\\n# mode: memory\\n' > "+raw)) + publisher := publish.NewFakePublisher() + publisher.On("Do").Return(nil) + return &austinPythonManager{commander: commander, publisher: publisher}, publisher, j, raw + } + + t.Run("raw output is annotated and published", func(t *testing.T) { + m, publisher, j, raw := setup(t, api.Raw) + + err, _ := m.invoke(j, "42") + require.NoError(t, err) + assert.Equal(t, 1, publisher.On("Do").InvokedTimes()) + content, err := os.ReadFile(raw) + require.NoError(t, err) + assert.Contains(t, string(content), "# no memory growth observed during the 30s window") + }) + + t.Run("flamegraph output fails with the reason", func(t *testing.T) { + m, publisher, j, _ := setup(t, api.FlameGraph) + + err, _ := m.invoke(j, "42") + require.Error(t, err) + assert.Contains(t, err.Error(), "no memory growth observed during the 30s window") + assert.Equal(t, 0, publisher.On("Do").InvokedTimes()) + }) +} diff --git a/test/e2e/python-austin.sh b/test/e2e/python-austin.sh new file mode 100755 index 0000000..c798bed --- /dev/null +++ b/test/e2e/python-austin.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# End-to-end check of Python memory profiling (austin) against real targets. +# +# Usage: test/e2e/python-austin.sh +# +# Each target runs a small Python program in its own container. The agent +# runs from the profiler image with --pid=container:, which gives it +# the same view a debugger pod has: the target's PIDs, but its own +# filesystem. That split is what broke austin, so the targets deliberately +# use interpreters at paths the profiler image does not have (3.12), a shared +# libpython (Debian), and the same path as the profiler's own interpreter +# (3.14) — which must be read from the target, not from the profiler. +set -euo pipefail + +image=${1:?usage: $0 } +work=$(mktemp -d) +containers=() +cleanup() { + if [ ${#containers[@]} -gt 0 ]; then docker rm -f "${containers[@]}" >/dev/null 2>&1 || true; fi + rm -rf "$work" +} +trap cleanup EXIT + +# Grows by 64 KiB every 5 ms, resetting at ~128 MiB so the container stays small. +cat >"$work/grow.py" <<'PY' +import time +hold = [] +while True: + hold.append(bytearray(64 * 1024)) + if len(hold) >= 2000: + hold = [] + time.sleep(0.005) +PY +# Allocates once, then only reads. +cat >"$work/steady.py" <<'PY' +import time +hold = [bytearray(64 * 1024) for _ in range(500)] +while True: + sum(len(b) for b in hold) + time.sleep(0.01) +PY +chmod 0644 "$work"/*.py + +start_target() { # name image script + docker run -d --name "$1" -v "$work/$3:/app.py:ro" "$2" python /app.py >/dev/null + containers+=("$1") +} + +# profile : prints the decompressed raw profile, or the agent's error +# on stderr and returns 1. +profile() { + local agent file + agent=$(docker run -d --privileged --pid="container:$1" --entrypoint /app/agent "$image" \ + --target-container-id e2e --target-container-runtime containerd \ + --target-container-runtime-path /run/containerd --pid 1 --lang python \ + --profiling-tool austin --output-type raw --duration 5s \ + --grace-period-ending 30s --compressor-type gzip) + containers+=("$agent") + for _ in $(seq 1 60); do + docker logs "$agent" 2>&1 | grep -q '"type":"\(result\|error\)"' && break + sleep 1 + done + file=$(docker logs "$agent" 2>&1 | grep -o '"file":"[^"]*"' | cut -d'"' -f4 || true) + if [ -z "$file" ]; then + docker logs "$agent" >&2 + return 1 + fi + docker cp -q "$agent:$file" "$work/profile.gz" + gzip -dc "$work/profile.gz" +} + +start_target t-alpine-312 python:3.12-alpine grow.py +start_target t-debian-312 python:3.12-slim grow.py +start_target t-alpine-314 python:3.14-alpine grow.py +start_target t-steady python:3.12-slim steady.py +sleep 3 + +failed=0 +for t in t-alpine-312 t-debian-312 t-alpine-314; do + if ! out=$(profile "$t"); then + echo "FAIL $t: no profile"; failed=1; continue + fi + samples=$(grep -c '^P' <<<"$out" || true) + version=$(grep '^# python:' <<<"$out" || true) + want=$(docker exec "$t" python -c 'import platform; print(platform.python_version())') + if [ "$samples" -eq 0 ] || [ "$version" != "# python: $want" ]; then + echo "FAIL $t: samples=$samples, header '$version', want python $want"; failed=1 + else + echo "ok $t: $samples samples, python $want" + fi +done + +if ! out=$(profile t-steady); then + echo "FAIL t-steady: no profile"; failed=1 +elif grep -q '^# no memory growth observed' <<<"$out"; then + echo "ok t-steady: reported no memory growth" +else + echo "FAIL t-steady: missing the no-growth note"; failed=1 +fi + +exit "$failed" From bfcbf08b31661429ce67352978a50c15c666ca5e Mon Sep 17 00:00:00 2001 From: mayankpande88 Date: Sat, 3 Oct 2026 23:12:13 +0530 Subject: [PATCH 2/2] fix(python): copy austin's target files from the mapped inode, and clean up Address review: - read the interpreter through /proc//exe and libpython through /proc//map_files/ instead of /proc//root, so a file replaced on disk since the process started ("(deleted)" in maps) is read as mapped rather than from its replacement - remove the per-PID copy directory once austin exits - count samples on scanner.Bytes() instead of allocating each line --- internal/agent/profiler/python_austin.go | 93 +++++++++++++------ .../profiler/python_austin_output_test.go | 73 ++++++++++----- 2 files changed, 116 insertions(+), 50 deletions(-) diff --git a/internal/agent/profiler/python_austin.go b/internal/agent/profiler/python_austin.go index dad7b9f..557a6ac 100644 --- a/internal/agent/profiler/python_austin.go +++ b/internal/agent/profiler/python_austin.go @@ -54,21 +54,22 @@ const ( // determine the version of the Python interpreter") or, worse, our own // interpreter (a python:3.14 target would be read through our python3.14). // -// The bind can't come straight from /proc//root: the kernel refuses a -// bind whose source lives in another mount namespace (EINVAL). So each file -// is copied out through /proc//root first and the copy is bound, inside -// a private mount namespace so neither the mounts nor the shadowing of our -// own python leak into the rest of the agent (mojo2austin runs on it). +// The bind can't come straight from the target: the kernel refuses a bind +// whose source lives in another mount namespace (EINVAL). So each file is +// copied out first and the copy is bound, inside a private mount namespace +// so neither the mounts nor the shadowing of our own python leak into the +// rest of the agent (mojo2austin runs on it). // -// Positional args: ... . +// Positional args: ... +// . const austinInTargetFSScript = `set -e -pid=$1; n=$2; shift 2 +dir=$1; n=$2; shift 2 i=0 while [ "$i" -lt "$n" ]; do - p=$1; shift - c="/tmp/austin-target-$pid$p" + src=$1; p=$2; shift 2 + c="$dir$p" mkdir -p "$(dirname "$c")" "$(dirname "$p")" - cp "/proc/$pid/root$p" "$c" + cp "$src" "$c" [ -e "$p" ] || touch "$p" mount --bind "$c" "$p" i=$((i+1)) @@ -82,28 +83,55 @@ var mojoMagic = []byte{'M', 'O', 'J', 3} // can point it at a fake tree. var procDir = "/proc" -var austinPythonCommand = func(commander executil.Commander, job *job.ProfilingJob, pid string, fileName string, targetFiles []string) *exec.Cmd { +// sourceReadable reports whether a target file can be copied out. A var +// because /proc//exe and map_files are magic links a fake tree can't +// reproduce: they resolve to the mapped inode, not to the path they print. +var sourceReadable = func(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +var austinPythonCommand = func(commander executil.Commander, job *job.ProfilingJob, pid string, fileName string, copyDir string, targetFiles []austinTargetFile) *exec.Cmd { interval := strconv.Itoa(int(job.Interval.Seconds())) austinArgs := []string{"-p", pid, "-o", fileName, "-x", interval, "-m"} if len(targetFiles) == 0 { return commander.Command(austinLocation, austinArgs...) } - args := []string{"-m", "--propagation", "private", "sh", "-c", austinInTargetFSScript, "sh", pid, strconv.Itoa(len(targetFiles))} - args = append(args, targetFiles...) + args := []string{"-m", "--propagation", "private", "sh", "-c", austinInTargetFSScript, "sh", copyDir, strconv.Itoa(len(targetFiles))} + for _, f := range targetFiles { + args = append(args, f.Source, f.Path) + } args = append(args, austinArgs...) return commander.Command(unshareLocation, args...) } +// austinTargetFile is one file austin opens in the target: Path is where +// austin looks for it, Source where the exact file the process mapped can be +// read from this container. +type austinTargetFile struct { + Path string + Source string +} + // austinTargetFiles returns the files austin opens by the path it finds in // /proc//maps: the interpreter binary and, for a shared build, -// libpython. Only files reachable through /proc//root are returned, so -// the caller can mount every one of them. -func austinTargetFiles(pid string) ([]string, error) { +// libpython. +// +// Each is read through a magic link to the inode the process actually +// mapped (/proc//exe for the interpreter, /proc//map_files/ +// for libpython), not through /proc//root. A file replaced on disk +// since the process started (maps then says "(deleted)") would otherwise be +// read from its replacement, and austin would resolve its symbols against +// the wrong build. +func austinTargetFiles(pid string) ([]austinTargetFile, error) { exe, err := os.Readlink(filepath.Join(procDir, pid, "exe")) if err != nil { return nil, errors.Wrapf(err, "could not resolve the interpreter of PID %s", pid) } - candidates := []string{strings.TrimSuffix(exe, " (deleted)")} + candidates := []austinTargetFile{{ + Path: strings.TrimSuffix(exe, " (deleted)"), + Source: filepath.Join(procDir, pid, "exe"), + }} maps, err := os.ReadFile(filepath.Join(procDir, pid, "maps")) //nolint:gosec // path built from procDir and a numeric PID if err != nil { @@ -114,20 +142,23 @@ func austinTargetFiles(pid string) ([]string, error) { if len(fields) < 6 { continue } - path := strings.Join(fields[5:], " ") + path := strings.TrimSuffix(strings.Join(fields[5:], " "), " (deleted)") if strings.HasPrefix(filepath.Base(path), "libpython") { - candidates = append(candidates, path) + candidates = append(candidates, austinTargetFile{ + Path: path, + Source: filepath.Join(procDir, pid, "map_files", fields[0]), + }) } } - var files []string + var files []austinTargetFile seen := map[string]bool{} for _, f := range candidates { - if seen[f] || !filepath.IsAbs(f) { + if seen[f.Path] || !filepath.IsAbs(f.Path) { continue } - seen[f] = true - if _, err := os.Stat(filepath.Join(procDir, pid, "root", f)); err != nil { + seen[f.Path] = true + if !sourceReadable(f.Source) { continue } files = append(files, f) @@ -135,6 +166,12 @@ func austinTargetFiles(pid string) ([]string, error) { return files, nil } +// austinCopyDir is where the target's interpreter files are copied for one +// PID. +func austinCopyDir(pid string) string { + return filepath.Join(common.TmpDir(), "austin-target-"+pid) +} + type AustinPythonProfiler struct { targetPIDs []string delay time.Duration @@ -218,10 +255,14 @@ func (p *austinPythonManager) invoke(job *job.ProfilingJob, pid string) (error, log.DebugLogLn(err.Error()) } binary := "unknown binary" + copyDir := austinCopyDir(pid) if len(targetFiles) > 0 { - binary = targetFiles[0] + binary = targetFiles[0].Path + // The copies are bound only inside austin's own mount namespace, + // which is gone once it exits; libpython alone can be tens of MB. + defer func() { _ = os.RemoveAll(copyDir) }() } - cmd := austinPythonCommand(p.commander, job, pid, fileName, targetFiles) + cmd := austinPythonCommand(p.commander, job, pid, fileName, copyDir, targetFiles) cmd.Stdout = &out cmd.Stderr = &stderr err = cmd.Run() @@ -334,7 +375,7 @@ func austinSampleCount(fileName string) (int, error) { scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) for scanner.Scan() { - if strings.HasPrefix(scanner.Text(), "P") { + if line := scanner.Bytes(); len(line) > 0 && line[0] == 'P' { n++ } } diff --git a/internal/agent/profiler/python_austin_output_test.go b/internal/agent/profiler/python_austin_output_test.go index 4a9c5f9..8d84746 100644 --- a/internal/agent/profiler/python_austin_output_test.go +++ b/internal/agent/profiler/python_austin_output_test.go @@ -84,19 +84,13 @@ func Test_convertMojo(t *testing.T) { }) } -// fakeProc builds a /proc/ tree: an exe link, a maps file, and the -// files present under the target's root. -func fakeProc(t *testing.T, pid, exe, maps string, rootFiles ...string) string { +// fakeProc builds a /proc/ tree with an exe link and a maps file. +func fakeProc(t *testing.T, pid, exe, maps string) string { t.Helper() dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, pid, "root"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, pid), 0o755)) require.NoError(t, os.Symlink(exe, filepath.Join(dir, pid, "exe"))) require.NoError(t, os.WriteFile(filepath.Join(dir, pid, "maps"), []byte(maps), 0o600)) - for _, f := range rootFiles { - p := filepath.Join(dir, pid, "root", f) - require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) - require.NoError(t, os.WriteFile(p, nil, 0o600)) - } return dir } @@ -107,8 +101,21 @@ func useProcDir(t *testing.T, dir string) { t.Cleanup(func() { procDir = old }) } +// readableSources makes only the given sources (relative to dir) readable. +func readableSources(t *testing.T, dir string, rel ...string) { + t.Helper() + ok := map[string]bool{} + for _, r := range rel { + ok[filepath.Join(dir, r)] = true + } + old := sourceReadable + sourceReadable = func(path string) bool { return ok[path] } + t.Cleanup(func() { sourceReadable = old }) +} + // Test_austinTargetFiles — austin opens the interpreter and libpython by the -// paths in /proc//maps, so those are exactly the files to mount. +// paths in /proc//maps, so those are the files to mount, each read +// through the link to the inode the process actually mapped. func Test_austinTargetFiles(t *testing.T) { const maps = `55d0c0a00000-55d0c0a01000 r--p 00000000 00:2e 1234 /usr/local/bin/python3.12 7f1c40000000-7f1c40200000 r--p 00000000 00:2e 1235 /usr/local/lib/libpython3.12.so.1.0 @@ -117,30 +124,43 @@ func Test_austinTargetFiles(t *testing.T) { 7f1c60000000-7f1c60100000 r-xp 00000000 00:2e 1237 /usr/local/lib/python3.12/lib-dynload/_json.cpython-312-x86_64-linux-musl.so 7ffd10000000-7ffd10021000 rw-p 00000000 00:00 0 [stack] ` + libpython := "42/map_files/7f1c40000000-7f1c40200000" - t.Run("interpreter first, then libpython, each once", func(t *testing.T) { - useProcDir(t, fakeProc(t, "42", "/usr/local/bin/python3.12", maps, - "/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0", "/lib/ld-musl-x86_64.so.1")) + t.Run("interpreter first, then libpython from its first mapping", func(t *testing.T) { + dir := fakeProc(t, "42", "/usr/local/bin/python3.12", maps) + useProcDir(t, dir) + readableSources(t, dir, "42/exe", libpython) files, err := austinTargetFiles("42") require.NoError(t, err) - assert.Equal(t, []string{"/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0"}, files) + assert.Equal(t, []austinTargetFile{ + {Path: "/usr/local/bin/python3.12", Source: filepath.Join(dir, "42/exe")}, + {Path: "/usr/local/lib/libpython3.12.so.1.0", Source: filepath.Join(dir, libpython)}, + }, files) }) - t.Run("a file missing from the target's root is skipped", func(t *testing.T) { - useProcDir(t, fakeProc(t, "42", "/usr/local/bin/python3.12", maps, "/usr/local/bin/python3.12")) + t.Run("a source that can't be read is skipped", func(t *testing.T) { + dir := fakeProc(t, "42", "/usr/local/bin/python3.12", maps) + useProcDir(t, dir) + readableSources(t, dir, "42/exe") files, err := austinTargetFiles("42") require.NoError(t, err) - assert.Equal(t, []string{"/usr/local/bin/python3.12"}, files) + assert.Equal(t, []austinTargetFile{{Path: "/usr/local/bin/python3.12", Source: filepath.Join(dir, "42/exe")}}, files) }) - t.Run("a replaced interpreter keeps its path", func(t *testing.T) { - useProcDir(t, fakeProc(t, "42", "/usr/bin/python3.11 (deleted)", "", "/usr/bin/python3.11")) + t.Run("files replaced on disk keep their path and are read from the mapped inode", func(t *testing.T) { + const deleted = "7f1c40000000-7f1c40200000 r--p 00000000 00:2e 1235 /usr/lib/libpython3.11.so.1.0 (deleted)\n" + dir := fakeProc(t, "42", "/usr/bin/python3.11 (deleted)", deleted) + useProcDir(t, dir) + readableSources(t, dir, "42/exe", libpython) files, err := austinTargetFiles("42") require.NoError(t, err) - assert.Equal(t, []string{"/usr/bin/python3.11"}, files) + assert.Equal(t, []austinTargetFile{ + {Path: "/usr/bin/python3.11", Source: filepath.Join(dir, "42/exe")}, + {Path: "/usr/lib/libpython3.11.so.1.0", Source: filepath.Join(dir, libpython)}, + }, files) }) t.Run("an unknown PID is an error", func(t *testing.T) { @@ -155,16 +175,21 @@ func Test_austinPythonCommand(t *testing.T) { j := &job.ProfilingJob{Interval: 30 * time.Second} t.Run("no target files runs austin directly", func(t *testing.T) { - cmd := austinPythonCommand(executil.NewCommander(), j, "42", "/tmp/out", nil) + cmd := austinPythonCommand(executil.NewCommander(), j, "42", "/tmp/out", "/tmp/copies", nil) assert.Equal(t, []string{"austin", "-p", "42", "-o", "/tmp/out", "-x", "30", "-m"}, cmd.Args) }) t.Run("target files run austin in a private mount namespace", func(t *testing.T) { - files := []string{"/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0"} - cmd := austinPythonCommand(executil.NewCommander(), j, "42", "/tmp/out", files) + files := []austinTargetFile{ + {Path: "/usr/local/bin/python3.12", Source: "/proc/42/exe"}, + {Path: "/usr/local/lib/libpython3.12.so.1.0", Source: "/proc/42/map_files/7f1c40000000-7f1c40200000"}, + } + cmd := austinPythonCommand(executil.NewCommander(), j, "42", "/tmp/out", "/tmp/copies", files) assert.Equal(t, []string{ "unshare", "-m", "--propagation", "private", "sh", "-c", austinInTargetFSScript, "sh", - "42", "2", "/usr/local/bin/python3.12", "/usr/local/lib/libpython3.12.so.1.0", + "/tmp/copies", "2", + "/proc/42/exe", "/usr/local/bin/python3.12", + "/proc/42/map_files/7f1c40000000-7f1c40200000", "/usr/local/lib/libpython3.12.so.1.0", "-p", "42", "-o", "/tmp/out", "-x", "30", "-m", }, cmd.Args) })