From 303270c4e36018f4acc497a6bbd7b20d9d3d3cb7 Mon Sep 17 00:00:00 2001 From: nareshns2004 Date: Thu, 1 Oct 2026 20:28:56 +0530 Subject: [PATCH] rebuilding into kptk kernel performance analysis toolkit --- .github/workflows/ci.yml | 22 + .gitignore | 12 + CHANGELOG.md | 38 ++ LICENSE | 201 +++++++++ Makefile | 48 ++- README.md | 154 ++++--- benchmarks/c/false_sharing.c | 54 +++ benchmarks/c/mem_latency.c | 69 ++++ benchmarks/c/page_fault_cost.c | 62 +++ benchmarks/c/wakeup_latency.c | 70 ++++ benchmarks/cache_benchmark.c | 13 - benchmarks/cpu_stress.c | 11 - benchmarks/memory_stress.c | 14 - benchmarks/numa_benchmark.c | 14 - benchmarks/page_fault_generator.c | 14 - configs/benchmark_profiles.yaml | 6 - configs/default.yaml | 9 - configs/thresholds.example.json | 11 + docs/architecture.md | 84 ++++ docs/design-decisions.md | 62 +++ docs/examples.md | 82 ++++ docs/linux-performance.md | 81 ++++ docs/metrics.md | 66 +++ images/architecture.png | 0 images/cache.png | 0 images/numa.png | 0 images/sample_dashboard.png | 0 pyproject.toml | 54 +++ requirements.txt | 5 +- samples/laptop_profile.jsonl.gz | Bin 0 -> 12345 bytes samples/laptop_profile_report.md | 152 +++++++ samples/numa_misplaced.jsonl.gz | Bin 0 -> 8094 bytes samples/numa_misplaced_report.json | 627 +++++++++++++++++++++++++++++ samples/numa_misplaced_report.md | 222 ++++++++++ samples/parse_report.json | 85 ++++ samples/report.json | 66 --- scripts/benchmark_all.sh | 6 - scripts/collect_metrics.sh | 9 - scripts/install_dependencies.sh | 6 - scripts/run_microbenchmarks.sh | 12 + scripts/setup_hugepages.sh | 19 +- src/cache_analysis.py | 15 - src/hugepages.py | 14 - src/kptk/__init__.py | 9 + src/kptk/__main__.py | 3 + src/kptk/analysis/__init__.py | 1 + src/kptk/analysis/cache.py | 115 ++++++ src/kptk/analysis/cpu.py | 179 ++++++++ src/kptk/analysis/engine.py | 191 +++++++++ src/kptk/analysis/findings.py | 91 +++++ src/kptk/analysis/hugepages.py | 132 ++++++ src/kptk/analysis/memory.py | 128 ++++++ src/kptk/analysis/numa.py | 183 +++++++++ src/kptk/analysis/window.py | 53 +++ src/kptk/cli.py | 249 ++++++++++++ src/kptk/compare.py | 127 ++++++ src/kptk/parsers/__init__.py | 1 + src/kptk/parsers/proc.py | 246 +++++++++++ src/kptk/parsers/sysfs.py | 37 ++ src/kptk/parsers/tools.py | 102 +++++ src/kptk/perf_event.py | 266 ++++++++++++ src/kptk/profile.py | 64 +++ src/kptk/record.py | 278 +++++++++++++ src/kptk/report.py | 100 +++++ src/kptk/synth.py | 387 ++++++++++++++++++ src/kptk/topology.py | 148 +++++++ src/numa_analysis.py | 11 - src/page_faults.py | 11 - src/parser.py | 100 ----- src/perf_runner.py | 54 --- src/report.py | 40 -- src/scheduler_latency.py | 11 - src/utils.py | 19 - tests/test_analysis.py | 111 +++++ tests/test_cli.py | 76 ++++ tests/test_metrics.py | 17 - tests/test_parser.py | 26 -- tests/test_parsers.py | 131 ++++++ tests/test_perf_event.py | 64 +++ tests/test_record_topology.py | 57 +++ tests/test_reports.py | 14 - 81 files changed, 5757 insertions(+), 564 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 benchmarks/c/false_sharing.c create mode 100644 benchmarks/c/mem_latency.c create mode 100644 benchmarks/c/page_fault_cost.c create mode 100644 benchmarks/c/wakeup_latency.c delete mode 100644 benchmarks/cache_benchmark.c delete mode 100644 benchmarks/cpu_stress.c delete mode 100644 benchmarks/memory_stress.c delete mode 100644 benchmarks/numa_benchmark.c delete mode 100644 benchmarks/page_fault_generator.c delete mode 100644 configs/benchmark_profiles.yaml delete mode 100644 configs/default.yaml create mode 100644 configs/thresholds.example.json create mode 100644 docs/design-decisions.md delete mode 100644 images/architecture.png delete mode 100644 images/cache.png delete mode 100644 images/numa.png delete mode 100644 images/sample_dashboard.png create mode 100644 pyproject.toml create mode 100644 samples/laptop_profile.jsonl.gz create mode 100644 samples/laptop_profile_report.md create mode 100644 samples/numa_misplaced.jsonl.gz create mode 100644 samples/numa_misplaced_report.json create mode 100644 samples/numa_misplaced_report.md create mode 100644 samples/parse_report.json delete mode 100644 samples/report.json delete mode 100755 scripts/benchmark_all.sh delete mode 100755 scripts/collect_metrics.sh delete mode 100755 scripts/install_dependencies.sh create mode 100755 scripts/run_microbenchmarks.sh delete mode 100644 src/cache_analysis.py delete mode 100644 src/hugepages.py create mode 100644 src/kptk/__init__.py create mode 100644 src/kptk/__main__.py create mode 100644 src/kptk/analysis/__init__.py create mode 100644 src/kptk/analysis/cache.py create mode 100644 src/kptk/analysis/cpu.py create mode 100644 src/kptk/analysis/engine.py create mode 100644 src/kptk/analysis/findings.py create mode 100644 src/kptk/analysis/hugepages.py create mode 100644 src/kptk/analysis/memory.py create mode 100644 src/kptk/analysis/numa.py create mode 100644 src/kptk/analysis/window.py create mode 100644 src/kptk/cli.py create mode 100644 src/kptk/compare.py create mode 100644 src/kptk/parsers/__init__.py create mode 100644 src/kptk/parsers/proc.py create mode 100644 src/kptk/parsers/sysfs.py create mode 100644 src/kptk/parsers/tools.py create mode 100644 src/kptk/perf_event.py create mode 100644 src/kptk/profile.py create mode 100644 src/kptk/record.py create mode 100644 src/kptk/report.py create mode 100644 src/kptk/synth.py create mode 100644 src/kptk/topology.py delete mode 100644 src/numa_analysis.py delete mode 100644 src/page_faults.py delete mode 100644 src/parser.py delete mode 100644 src/perf_runner.py delete mode 100644 src/report.py delete mode 100644 src/scheduler_latency.py delete mode 100644 src/utils.py create mode 100644 tests/test_analysis.py create mode 100644 tests/test_cli.py delete mode 100644 tests/test_metrics.py delete mode 100644 tests/test_parser.py create mode 100644 tests/test_parsers.py create mode 100644 tests/test_perf_event.py create mode 100644 tests/test_record_topology.py delete mode 100644 tests/test_reports.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..63bd30e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: ci +on: + push: + branches: [main] + pull_request: +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: pip install -e ".[dev]" + - run: ruff check src tests && ruff format --check src tests + - run: mypy + - run: pytest -q + - run: make bench-build # compile-only; benchmarks are not executed in CI + - run: kptk demo numa_misplaced > /dev/null && kptk inspect diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ca5a759 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.venv/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +/recording*.jsonl* +/report*.md +/report*.json diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b84ef20 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,38 @@ +# Changelog + +## 0.2.0: analysis toolkit + +Rebuilt from the 0.1 text-to-JSON scaffold into a packaged toolkit (`kptk`). + +### Added +- **Recorder** that captures raw /proc and /sys text into portable JSONL recordings, with + deadline scheduling, per-sample cost tracking, and rate-limited page-table-walking reads. +- **perf_event_open via ctypes**: hardware/software/cache events, multiplexing correction, + and the fork → attach → exec (`enable_on_exec`) handshake for `kptk profile`. +- **Analyzers** for CPU scheduling (schedstat run-queue delay, PSI, voluntary/involuntary + switches, steal, imbalance), memory (faults, direct reclaim, swap, PSI), NUMA (allocation vs + access locality, numa_maps placement vs thread residency, SLIT distance, node exhaustion), + cache/TLB (IPC, MPKI, multiplexing), huge pages (THP mode, fallback, compaction, coverage, + idle hugetlb), and a config audit. +- Findings engine with severities, evidence and recommendations; USE summary; sparklines; + data-quality report; tunable thresholds (JSON); CI gate (`--fail-on`). +- `kptk compare`: multi-run A/B with a permutation test. +- Topology reconstruction (SMT, sockets, NUMA nodes, cache hierarchy). +- Synthetic two-socket recordings (`kptk demo`) and a test suite asserting exact findings per scenario. +- C microbenchmarks: pointer-chase latency (4K vs THP, NUMA), page-fault cost, wakeup latency, false sharing. +- Docs: architecture, metric reference, kernel mechanics, workflows, design decisions, interview guide. CI workflow. + +### Changed +- NUMA "remote" no longer uses `numa_miss` (a policy counter); locality uses + `local_node`/`other_node` and hinting faults. +- "Scheduler latency" now measures run-queue delay rather than context-switch counts. +- `perf stat` parsing keeps the multiplexing percentage. The v0.1 workflow is now `kptk parse`. +- Code is an installable package (`src/kptk`) instead of flat scripts with `sys.path` edits; + `parser.py` no longer shadows a standard-library module name. +- Configs are JSON (stdlib-only). LICENSE was empty and is now Apache-2.0. + +### Removed +- Empty placeholder images and docs; trivial C programs replaced by the microbenchmarks above. + +## 0.1.0 +- Initial scaffold. diff --git a/LICENSE b/LICENSE index e69de29..261eeb9 100644 --- a/LICENSE +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile index f899556..cf6f116 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,51 @@ -.PHONY: test sample-report benchmarks clean - +PYTHON ?= python3 CC ?= cc -CFLAGS ?= -O2 -Wall -Wextra -std=c11 +CFLAGS ?= -O2 -Wall -Wextra -std=c11 -pthread BUILD := build -BENCHMARKS := cache_benchmark cpu_stress memory_stress numa_benchmark page_fault_generator +BENCH := mem_latency page_fault_cost wakeup_latency false_sharing +export PYTHONPATH := src + +.PHONY: install dev test lint typecheck fmt check demo samples bench-build clean + +install: + $(PYTHON) -m pip install -e . + +dev: + $(PYTHON) -m pip install -e ".[dev]" test: - python3 -m pytest -q + $(PYTHON) -m pytest -q + +lint: + ruff check src tests + ruff format --check src tests + +typecheck: + mypy + +fmt: + ruff check --fix src tests + ruff format src tests + +check: lint typecheck test + +demo: + $(PYTHON) -m kptk demo numa_misplaced -sample-report: - python3 src/perf_runner.py --perf-file samples/perf_output.txt --vmstat-file samples/vmstat.txt --numastat-file samples/numastat.txt --output samples/report.json +samples: + $(PYTHON) -m kptk demo numa_misplaced --save samples/numa_misplaced.jsonl.gz --md samples/numa_misplaced_report.md --json samples/numa_misplaced_report.json + $(PYTHON) -m kptk parse --perf-file samples/perf_output.txt --vmstat-file samples/vmstat.txt --numastat-file samples/numastat.txt -o samples/parse_report.json -benchmarks: $(BENCHMARKS:%=$(BUILD)/%) +# Microbenchmarks are compiled here but never run by `make check`: they are +# deliberately CPU/memory heavy. Run them by hand on the machine under study. +bench-build: $(BENCH:%=$(BUILD)/%) $(BUILD): mkdir -p $@ -$(BUILD)/%: benchmarks/%.c | $(BUILD) +$(BUILD)/%: benchmarks/c/%.c | $(BUILD) $(CC) $(CFLAGS) $< -o $@ clean: - rm -rf $(BUILD) + rm -rf $(BUILD) .pytest_cache .mypy_cache .ruff_cache src/*.egg-info + find . -name __pycache__ -type d -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 7b9ded6..9d584c5 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,132 @@ -# Kernel Performance Toolkit +# Kernel Performance Toolkit (`kptk`) -A small, dependency-free toolkit for turning Linux `perf stat`, `vmstat`, and -`numastat` output into a single JSON report. It works in two modes: +**A Linux kernel performance analysis toolkit for CPU scheduling, memory behaviour, NUMA +locality, cache efficiency, page faults and huge pages.** It explains *why* a workload is slow, +with evidence from kernel counters and a concrete fix. -- **Offline analysis** parses saved command output. This is portable and needs - no special permissions. -- **Live collection** runs a command under `perf stat` and captures a `vmstat` - and `numastat` snapshot when those tools are available. +- **Zero dependencies**: Python standard library only; safe to drop onto a production host. +- **Low overhead**: records raw `/proc` and `/sys` text (~1.4 ms per sample measured on a laptop) and analyses later, anywhere. +- **Hardware counters without `perf`**: calls `perf_event_open(2)` directly via ctypes, with multiplexing correction. +- **Correct kernel semantics**: run-queue delay instead of "CPU %", allocation vs access locality for NUMA, THP fallback and compaction instead of just "THP on". -## Quick start +``` +$ kptk demo numa_misplaced -Generate a report from the bundled fixtures: +### 🔴 About 85% of the process's memory accesses are likely remote: threads ran mostly on node 0, memory sits mostly on node 1 +Evidence: `cpu_node_residency={0: 1}`, `memory_by_node={0: 0.15, 1: 0.85}`, `placement_mismatch=0.85`, `expected_slit_distance=19.4` +Action: Start it with numactl --cpunodebind=1 --membind=1, or migratepages 4242 1 0. -```bash -make sample-report -python3 -m json.tool samples/report.json +### 🟠 Likely memory-bound: IPC 0.60, LLC 17.9 misses per 1k instructions; anonymous working set 49152 MiB vs 32 MiB L3 +### 🟠 dTLB miss rate 3.0% (10.00 per 1k instructions): page walks are significant +### 🟠 THP enabled=always with defrag=always: page faults may stall on synchronous compaction +### 🟠 44% of huge-page faults fell back to 4 KiB pages +... ``` +Synthetic two-socket scenario. Full report: [`samples/numa_misplaced_report.md`](samples/numa_misplaced_report.md); +a real laptop recording is in [`samples/laptop_profile_report.md`](samples/laptop_profile_report.md). -Run the test suite: +## What it measures -```bash -python3 -m pytest -q -``` +| Domain | Key signals | Example findings | +|---|---|---| +| **CPU scheduling** | `/proc/schedstat` run-queue delay, per-task wait share, PSI cpu, voluntary vs involuntary switches, steal, per-CPU imbalance | CPU saturation; process waits 45% of runnable time in run queues; preemption storm | +| **Memory and faults** | minor/major fault rates, direct reclaim (`allocstall`), swap, PSI memory, MemAvailable | Direct reclaim stalls; major faults; minor-fault storm from allocator churn | +| **NUMA locality** | `local_node`/`other_node`, AutoNUMA hinting-fault locality, `numa_maps` placement vs thread residency, SLIT distances, per-node free memory | Process misplaced across nodes; node exhausted, so allocations spill remote | +| **Cache efficiency** | IPC, LLC MPKI, branch and L1D miss rates, multiplexing coverage | Likely memory-bound (working set vs LLC); bad speculation | +| **TLB and huge pages** | dTLB MPKI, THP mode and defrag, fallback %, compaction stalls, heap THP coverage, idle hugetlb pools | Sync compaction risk; THP fallback (fragmentation); low coverage with TLB pressure | +| **Configuration** | `zone_reclaim_mode`, governors, `numa_balancing`, `perf_event_paranoid` | Risky sysctls, profiling blocked | -Build the small workload fixtures (a C compiler is required): +Every report also has a **USE table** (utilisation / saturation / errors per resource), +**sparklines** over time, and a **data-quality** section listing anything the kernel didn't expose. + +## Quick start ```bash -make benchmarks +pip install -e . # or just: PYTHONPATH=src python3 -m kptk ... + +kptk inspect # topology, caches, THP/hugetlb, sysctl audit +kptk demo --list && kptk demo cpu_saturated # synthetic two-socket scenarios +kptk profile -- ./my_workload --args # counters + recording for one command +kptk record -p -d 60 -o svc.jsonl.gz # watch a running process +kptk analyze svc.jsonl.gz --md report.md --json report.json --fail-on warning +kptk compare --before a*.json --after b*.json # permutation-tested A/B ``` -## Usage +More workflows, including the CI gate and a full case study: [docs/examples.md](docs/examples.md). -Parse existing outputs: +## How it works -```bash -python3 src/perf_runner.py \ - --perf-file samples/perf_output.txt \ - --vmstat-file samples/vmstat.txt \ - --numastat-file samples/numastat.txt \ - --output report.json --markdown report.md +```mermaid +flowchart LR + P["/proc + /sys"] -->|verbatim text| R[Recorder] --> J[("recording.jsonl.gz")] + K["perf_event_open
(ctypes)"] -->|"fork → attach → exec"| J + J --> E["pure analysis
cpu · memory · numa · cache · hugepages · config"] + E --> F["findings + USE + time series"] --> O["Markdown / JSON / CI exit code"] + F --> C["compare (permutation test)"] ``` -Collect metrics while a program runs: +* **Capture, then analyse** ([ADR-1](docs/design-decisions.md)): the recorder never parses, so + recordings are portable, re-analysable and cheap to take. +* **Exact counting from the first instruction** (ADR-4): fork, child blocks before exec, + parent attaches counters with `enable_on_exec`, then releases it (the `perf stat` handshake). +* **Expensive files are rate-limited** (ADR-8): `numa_maps`/`smaps_rollup` walk the target's + page tables, so they're read every N samples. -```bash -python3 src/perf_runner.py --command ./my-workload --size 1024 +Details: [architecture](docs/architecture.md) · [metric reference](docs/metrics.md) · +[kernel mechanics](docs/linux-performance.md) · [design decisions](docs/design-decisions.md) + +## Microbenchmarks + +`benchmarks/c/` reproduces each phenomenon on real hardware. They're built by `make bench-build` +but never run in CI, because they are deliberately heavy: + +| Benchmark | Shows | +|---|---| +| `mem_latency [--huge]` | Pointer-chase latency vs working set: L1/L2/L3/DRAM cliffs, and the TLB-reach shift with huge pages; under `numactl` the remote-node penalty | +| `page_fault_cost [MiB]` | First-touch cost: base pages vs `MAP_POPULATE` vs THP vs pre-faulted | +| `wakeup_latency [n cpuA cpuB]` | Block/wake latency percentiles; same core vs same socket vs cross-socket | +| `false_sharing [threads]` | Cache-line ping-pong: packed vs padded per-thread counters | + +## Project layout + +``` +src/kptk/ + parsers/ proc.py · sysfs.py · tools.py (perf stat, vmstat, numastat text) + analysis/ cpu · memory · numa · cache · hugepages · engine · findings · window + record.py Source abstraction, deadline-scheduled Recorder, JSONL recordings + perf_event.py perf_event_open via ctypes (attr struct, encodings, multiplex scaling) + profile.py fork → attach → exec profiling + topology.py CPUs, SMT, sockets, NUMA nodes/distances, cache hierarchy + compare.py multi-run A/B with permutation test + synth.py synthetic two-socket recordings (tests, demo) + report.py · cli.py +benchmarks/c/ mem_latency · page_fault_cost · wakeup_latency · false_sharing +samples/ real laptop recording, synthetic scenario, v0.1 text inputs, generated reports +docs/ architecture · metrics · linux-performance · examples · design-decisions · interview-guide ``` -For live collection, install the Linux `perf` tools and `numastat` (usually -provided by the `numactl` package). Some systems restrict performance counters; -adjust `kernel.perf_event_paranoid` or use an administrator-approved setup when -appropriate. No `sudo` is used by the toolkit itself. +## Requirements and permissions -## Report contents +Linux, Python ≥ 3.10. Everything except hardware counters works unprivileged. Counters need +`kernel.perf_event_paranoid ≤ 2` or `CAP_PERFMON`; without them the report says so and +continues. Run-queue latency needs `CONFIG_SCHEDSTATS`; PSI needs `CONFIG_PSI`. +`scripts/setup_hugepages.sh` reserves hugetlb pages and requires root. -The JSON report preserves the raw perf, vmstat, and NUMA inputs and derives: +## Development -- cache references, misses, miss rate, and instructions per miss; -- minor/major fault totals and major-fault percentage; -- context switches, CPU migrations, and switches per task-clock millisecond; -- local versus remote NUMA allocation percentage. +```bash +make check # ruff + mypy + pytest (43 tests, < 1 s; no CPU-heavy tests) +make bench-build # compile microbenchmarks (does not run them) +make samples # regenerate the sample reports +``` -Missing or unsupported counters are represented as zero-derived metrics, so a -report can still be compared across hosts with different capabilities. +## Limitations and roadmap -## Layout +- Counting, not sampling: no stacks or code attribution. Pair it with `perf record` or eBPF. +- Generic PMU events give a coarse "top-down lite"; next is Intel topdown slots / AMD equivalents. +- Process view is pid-level. Next: per-thread residency from `/proc//task/*`. +- Next: eBPF run-queue latency histograms, and cgroup v2 (`cpu.stat` throttling, per-cgroup PSI). -- `src/perf_runner.py` — command-line entry point. -- `src/parser.py` — parsers for human-readable and CSV perf output. -- `src/report.py` — report construction plus JSON/Markdown output. -- `samples/` — known-good offline inputs and a generated example report. -- `scripts/` — convenience collection and huge-page setup scripts. +## License -`scripts/setup_hugepages.sh` changes a kernel setting and intentionally requires -root. Run it only when huge pages are part of your workload configuration. +Apache-2.0. See [LICENSE](LICENSE). diff --git a/benchmarks/c/false_sharing.c b/benchmarks/c/false_sharing.c new file mode 100644 index 0000000..9fe5d7d --- /dev/null +++ b/benchmarks/c/false_sharing.c @@ -0,0 +1,54 @@ +/* + * false_sharing: N threads increment their *own* counters, packed vs padded. + * + * Packed counters share 64-byte cache lines, so every increment invalidates the + * line in the other cores' caches (MESI ping-pong), even though no data is + * logically shared. Padding each counter to its own line removes the coherence + * traffic. Expect a large slowdown for "packed" with >= 2 threads on different cores. + * + * ./false_sharing [threads=4] [iterations=100000000] + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#define MAX_THREADS 64 +struct padded { _Alignas(64) volatile uint64_t v; char pad[64 - sizeof(uint64_t)]; }; + +static volatile uint64_t packed[MAX_THREADS]; +static struct padded padded[MAX_THREADS]; +static long iters = 100000000; + +static void *bump_packed(void *arg) { + long id = (long)arg; + for (long i = 0; i < iters; i++) packed[id]++; + return NULL; +} + +static void *bump_padded(void *arg) { + long id = (long)arg; + for (long i = 0; i < iters; i++) padded[id].v++; + return NULL; +} + +static double run(void *(*fn)(void *), int n) { + pthread_t t[MAX_THREADS]; + struct timespec t0, t1; + clock_gettime(CLOCK_MONOTONIC, &t0); + for (long i = 0; i < n; i++) pthread_create(&t[i], NULL, fn, (void *)i); + for (int i = 0; i < n; i++) pthread_join(t[i], NULL); + clock_gettime(CLOCK_MONOTONIC, &t1); + return (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6; +} + +int main(int argc, char **argv) { + int n = argc > 1 ? atoi(argv[1]) : 4; + if (n > MAX_THREADS) n = MAX_THREADS; + if (argc > 2) iters = atol(argv[2]); + double a = run(bump_packed, n), b = run(bump_padded, n); + printf("threads=%d packed=%.1fms padded=%.1fms slowdown=%.1fx\n", n, a, b, a / b); + return 0; +} diff --git a/benchmarks/c/mem_latency.c b/benchmarks/c/mem_latency.c new file mode 100644 index 0000000..333d457 --- /dev/null +++ b/benchmarks/c/mem_latency.c @@ -0,0 +1,69 @@ +/* + * mem_latency: load-to-use latency vs working-set size, by pointer chasing. + * + * Each element stores the index of the next one, in a random cyclic permutation + * (Sattolo's algorithm), so every load depends on the previous one: no + * memory-level parallelism, and the hardware prefetcher can't guess the next + * address. ns/load vs size shows the L1 -> L2 -> L3 -> DRAM cliffs, and with + * --huge the TLB-reach difference between 4 KiB and 2 MiB pages. + * + * ./mem_latency # 4 KiB .. 1 GiB, base pages + * ./mem_latency --huge # same, madvise(MADV_HUGEPAGE) + * numactl --cpunodebind=0 --membind=1 ./mem_latency # remote-node latency + * + * Output: CSV size_bytes,ns_per_load,huge + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static uint64_t rng_state = 0x9E3779B97F4A7C15ull; +static uint64_t xorshift(void) { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 7; + rng_state ^= rng_state << 17; + return rng_state; +} + +static double now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1e9 + ts.tv_nsec; +} + +static double chase(size_t bytes, int huge, size_t min_loads) { + size_t n = bytes / sizeof(uint64_t); + /* One cache line per element would hide spatial locality; one u64 per element + with a random permutation already defeats it while keeping the size exact. */ + uint64_t *a = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (a == MAP_FAILED) return -1; + if (huge) madvise(a, bytes, MADV_HUGEPAGE); + for (size_t i = 0; i < n; i++) a[i] = i; + for (size_t i = n - 1; i > 0; i--) { /* Sattolo: a single cycle through all elements */ + size_t j = xorshift() % i; + uint64_t t = a[i]; a[i] = a[j]; a[j] = t; + } + size_t loads = n * 4 > min_loads ? n * 4 : min_loads; + volatile uint64_t p = 0; + for (size_t i = 0; i < n; i++) p = a[p]; /* warm up caches and TLB */ + double t0 = now_ns(); + for (size_t i = 0; i < loads; i++) p = a[p]; + double dt = now_ns() - t0; + munmap(a, bytes); + return dt / loads + 0 * (double)p; +} + +int main(int argc, char **argv) { + int huge = argc > 1 && strcmp(argv[1], "--huge") == 0; + size_t max_bytes = (size_t)1 << 30; + printf("size_bytes,ns_per_load,huge\n"); + for (size_t b = 4096; b <= max_bytes; b *= 2) { + printf("%zu,%.2f,%d\n", b, chase(b, huge, 20 * 1000 * 1000), huge); + fflush(stdout); + } + return 0; +} diff --git a/benchmarks/c/page_fault_cost.c b/benchmarks/c/page_fault_cost.c new file mode 100644 index 0000000..1521add --- /dev/null +++ b/benchmarks/c/page_fault_cost.c @@ -0,0 +1,62 @@ +/* + * page_fault_cost: what a first-touch page fault costs, and how to avoid paying it. + * + * Touches a fresh anonymous mapping one byte per 4 KiB in four modes: + * base - plain mmap, one minor fault per 4 KiB page + * populate - MAP_POPULATE, faults taken up front inside mmap() + * thp - madvise(MADV_HUGEPAGE), ideally one fault per 2 MiB + * prefault - touch once, then time a second pass (no faults: the floor) + * Fault counts come from getrusage(RUSAGE_SELF).ru_minflt. + * + * ./page_fault_cost [MiB=512] + * Output: CSV mode,mib,ms,minor_faults,ns_per_4k_page + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static double now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6; +} + +static long minflt(void) { + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); + return ru.ru_minflt; +} + +static void run(const char *mode, size_t mib) { + size_t bytes = mib << 20, pages = bytes / 4096; + int flags = MAP_PRIVATE | MAP_ANONYMOUS | (strcmp(mode, "populate") == 0 ? MAP_POPULATE : 0); + long f0 = minflt(); + double t0 = now_ms(); + char *p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, flags, -1, 0); + if (p == MAP_FAILED) { perror("mmap"); exit(1); } + if (strcmp(mode, "thp") == 0) madvise(p, bytes, MADV_HUGEPAGE); + if (strcmp(mode, "prefault") == 0) { + for (size_t i = 0; i < pages; i++) p[i * 4096] = 1; + f0 = minflt(); + t0 = now_ms(); + } + for (size_t i = 0; i < pages; i++) p[i * 4096] = (char)i; + double ms = now_ms() - t0; + long faults = minflt() - f0; + printf("%s,%zu,%.2f,%ld,%.1f\n", mode, mib, ms, faults, ms * 1e6 / pages); + munmap(p, bytes); +} + +int main(int argc, char **argv) { + size_t mib = argc > 1 ? strtoull(argv[1], NULL, 10) : 512; + printf("mode,mib,ms,minor_faults,ns_per_4k_page\n"); + run("base", mib); + run("populate", mib); + run("thp", mib); + run("prefault", mib); + return 0; +} diff --git a/benchmarks/c/wakeup_latency.c b/benchmarks/c/wakeup_latency.c new file mode 100644 index 0000000..5508d1e --- /dev/null +++ b/benchmarks/c/wakeup_latency.c @@ -0,0 +1,70 @@ +/* + * wakeup_latency: thread wakeup latency distribution via pipe ping-pong. + * + * Two threads bounce a byte through a pair of pipes; each round trip is two + * block/wake cycles through the scheduler. Optional pinning shows the effect of + * placement: same core (SMT sibling), same socket, or across sockets. + * Comparable to `perf bench sched pipe`, but reports percentiles, not just the mean. + * + * ./wakeup_latency [iterations=200000] [cpuA cpuB] + * Output: p50/p90/p99/p999/max one-way latency in ns. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static int ab[2], ba[2]; +static long iters = 200000; +static int cpu_b = -1; + +static void pin(int cpu) { + if (cpu < 0) return; + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +} + +static void *echo(void *arg) { + (void)arg; + pin(cpu_b); + char c; + for (long i = 0; i < iters; i++) { + if (read(ab[0], &c, 1) != 1 || write(ba[1], &c, 1) != 1) break; + } + return NULL; +} + +static int cmp(const void *a, const void *b) { + double x = *(const double *)a, y = *(const double *)b; + return (x > y) - (x < y); +} + +int main(int argc, char **argv) { + if (argc > 1) iters = atol(argv[1]); + int cpu_a = argc > 3 ? atoi(argv[2]) : -1; + cpu_b = argc > 3 ? atoi(argv[3]) : -1; + if (pipe(ab) || pipe(ba)) return 1; + pin(cpu_a); + pthread_t t; + pthread_create(&t, NULL, echo, NULL); + double *lat = malloc(sizeof(double) * iters); + char c = 'x'; + for (long i = 0; i < iters; i++) { + struct timespec t0, t1; + clock_gettime(CLOCK_MONOTONIC, &t0); + if (write(ab[1], &c, 1) != 1 || read(ba[0], &c, 1) != 1) break; + clock_gettime(CLOCK_MONOTONIC, &t1); + lat[i] = ((t1.tv_sec - t0.tv_sec) * 1e9 + (t1.tv_nsec - t0.tv_nsec)) / 2; /* one-way */ + } + pthread_join(t, NULL); + qsort(lat, iters, sizeof(double), cmp); + printf("cpus=%d,%d iterations=%ld\n", cpu_a, cpu_b, iters); + printf("p50=%.0fns p90=%.0fns p99=%.0fns p999=%.0fns max=%.0fns\n", lat[iters / 2], lat[iters * 9 / 10], lat[iters * 99 / 100], lat[iters * 999 / 1000], lat[iters - 1]); + free(lat); + return 0; +} diff --git a/benchmarks/cache_benchmark.c b/benchmarks/cache_benchmark.c deleted file mode 100644 index ee5f948..0000000 --- a/benchmarks/cache_benchmark.c +++ /dev/null @@ -1,13 +0,0 @@ -#include -#include -#include - -int main(int argc, char **argv) { - size_t count = argc > 1 ? strtoull(argv[1], NULL, 10) : 8 * 1024 * 1024; - uint64_t *data = calloc(count, sizeof(*data)); - if (!data) return 1; - for (size_t i = 0; i < count; i += 16) data[i] += i; - printf("value=%llu\n", (unsigned long long)data[count - 16]); - free(data); - return 0; -} diff --git a/benchmarks/cpu_stress.c b/benchmarks/cpu_stress.c deleted file mode 100644 index 6840022..0000000 --- a/benchmarks/cpu_stress.c +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include -#include - -int main(int argc, char **argv) { - long iterations = argc > 1 ? strtol(argv[1], NULL, 10) : 100000000L; - volatile uint64_t value = 1; - for (long i = 0; i < iterations; i++) value = value * 1664525u + 1013904223u; - printf("checksum=%llu\n", (unsigned long long)value); - return 0; -} diff --git a/benchmarks/memory_stress.c b/benchmarks/memory_stress.c deleted file mode 100644 index db50bfb..0000000 --- a/benchmarks/memory_stress.c +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include -#include - -int main(int argc, char **argv) { - size_t mib = argc > 1 ? strtoull(argv[1], NULL, 10) : 64; - size_t bytes = mib * 1024 * 1024; - char *buffer = malloc(bytes); - if (!buffer) return 1; - memset(buffer, 0x5a, bytes); - printf("allocated=%zu MiB\n", mib); - free(buffer); - return 0; -} diff --git a/benchmarks/numa_benchmark.c b/benchmarks/numa_benchmark.c deleted file mode 100644 index 9942ed4..0000000 --- a/benchmarks/numa_benchmark.c +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include -#include - -/* Allocation placement is controlled externally (for example, numactl). */ -int main(int argc, char **argv) { - size_t count = argc > 1 ? strtoull(argv[1], NULL, 10) : 16 * 1024 * 1024; - uint64_t *data = calloc(count, sizeof(*data)); - if (!data) return 1; - for (size_t i = 0; i < count; i += 512) data[i] = i; - printf("pages touched=%zu\n", count * sizeof(*data) / 4096); - free(data); - return 0; -} diff --git a/benchmarks/page_fault_generator.c b/benchmarks/page_fault_generator.c deleted file mode 100644 index 4b714a2..0000000 --- a/benchmarks/page_fault_generator.c +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include -#include - -int main(int argc, char **argv) { - size_t pages = argc > 1 ? strtoull(argv[1], NULL, 10) : 4096; - long page_size = sysconf(_SC_PAGESIZE); - char *memory = malloc(pages * (size_t)page_size); - if (!memory) return 1; - for (size_t i = 0; i < pages; ++i) memory[i * (size_t)page_size] = (char)i; - printf("touched=%zu pages\n", pages); - free(memory); - return 0; -} diff --git a/configs/benchmark_profiles.yaml b/configs/benchmark_profiles.yaml deleted file mode 100644 index c082f2e..0000000 --- a/configs/benchmark_profiles.yaml +++ /dev/null @@ -1,6 +0,0 @@ -cache: - command: ./build/cache_benchmark - events: cache-references,cache-misses,cycles,instructions -memory: - command: ./build/memory_stress - events: page-faults,minor-faults,major-faults,task-clock diff --git a/configs/default.yaml b/configs/default.yaml deleted file mode 100644 index b131f5d..0000000 --- a/configs/default.yaml +++ /dev/null @@ -1,9 +0,0 @@ -events: - - cycles - - instructions - - cache-references - - cache-misses - - context-switches - - cpu-migrations - - page-faults -duration_seconds: 10 diff --git a/configs/thresholds.example.json b/configs/thresholds.example.json new file mode 100644 index 0000000..208b31b --- /dev/null +++ b/configs/thresholds.example.json @@ -0,0 +1,11 @@ +{ + "cpu_psi_some_warn": 0.10, + "runq_wait_share_warn": 0.10, + "involuntary_switch_rate_warn": 1000, + "major_fault_rate_warn": 10, + "remote_alloc_warn": 0.10, + "numa_placement_mismatch_warn": 0.30, + "llc_mpki_warn": 5.0, + "dtlb_miss_warn": 0.01, + "thp_fallback_warn": 0.20 +} diff --git a/docs/architecture.md b/docs/architecture.md index e69de29..302e238 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -0,0 +1,84 @@ +# Architecture + +## Goals + +1. **Safe to run on production hosts**: no kernel modules, no eBPF, no `perf` binary, no + runtime dependencies, sub-millisecond-per-sample overhead, read-only. +2. **Correct kernel semantics**: every metric traces back to a documented kernel counter + and is interpreted the way the kernel defines it (see [metrics.md](metrics.md)). +3. **Actionable output**: findings with evidence and a concrete next step, not a wall of numbers. +4. **Reproducible analysis**: what was captured can be re-analysed anywhere, later. + +## Capture, then analyse + +```mermaid +flowchart LR + subgraph host["Host under study"] + P["/proc + /sys
text files"] -->|read verbatim| R["Recorder
deadline-scheduled"] + K["perf_event_open
(ctypes)"] -->|counters| PR["profile
fork → attach → exec"] + R --> PR + end + R --> J[("recording.jsonl.gz
meta · static · samples · counters")] + PR --> J + J --> E["analysis engine
(pure functions)"] + T["perf stat text
(optional)"] --> E + E --> D1[cpu] & D2[memory] & D3[numa] & D4[cache] & D5[hugepages] & D6[config] + D1 & D2 & D3 & D4 & D5 & D6 --> F["findings · USE table · time series"] + F --> O["Markdown / JSON report
CI gate (--fail-on)"] + F --> C["compare: permutation test
across runs"] +``` + +The recorder **never parses**. It copies file contents into the recording and timestamps +them. This split matters for three reasons: + +* **Overhead**: a sample is ~20-40 `open/read/close` calls, ~1 ms on a laptop and far below 1% + of a core at 1 s intervals. Parsing happens off the hot path. +* **Portability**: a recording from a locked-down production host can be analysed on a laptop, + attached to a ticket, and re-analysed after a parser fix or threshold change. +* **Testability**: analysis is a pure function `Recording -> Report`. Tests feed it synthetic + recordings (`kptk/synth.py`) of a two-socket server and assert exact findings. + +## Recording format + +JSONL (optionally gzipped), one object per line: + +| line | content | +|---|---| +| `{"type": "meta", ...}` | host, kernel, arch, `user_hz`, page size, interval, target pid/command, `static` files | +| `{"type": "sample", "ts", "cost_s", "files"}` | dynamic files at one instant, plus the recorder's own cost | +| `{"type": "counters", "events"}` | hardware/software counter totals (optional) | + +**Static** (captured once): CPU topology, cache hierarchy, NUMA cpulists and SLIT distances, +THP and khugepaged settings, hugetlb pools, relevant sysctls, cpufreq governors. +**Dynamic** (every sample): `/proc/{stat,schedstat,vmstat,meminfo,loadavg}`, PSI, per-node +`numastat` and `meminfo`, and for a target pid `stat`, `status`, `schedstat`. +**Slow** (every N samples): `/proc//{numa_maps,smaps_rollup}`. These walk the target's +page tables under its mmap lock, so reading them every sample would perturb the process being +measured. + +## Modules + +| Module | Role | +|---|---| +| `parsers/proc.py`, `parsers/sysfs.py` | Pure parsers for /proc and /sys formats | +| `parsers/tools.py` | `perf stat` (CSV + human, keeps multiplexing %), `vmstat`, `numastat` text | +| `record.py` | `Source` abstraction (live / dict), `Recorder`, `Recording` I/O | +| `perf_event.py` | `perf_event_open(2)` via ctypes: attr struct, event encoding, multiplex scaling | +| `profile.py` | fork → attach counters (`enable_on_exec`) → exec, plus recording until exit | +| `topology.py` | CPUs, SMT, packages, NUMA nodes and distances, cache hierarchy | +| `analysis/window.py` | First/last-sample deltas and rates | +| `analysis/{cpu,memory,numa,cache,hugepages}.py` | Domain analyzers → metrics + findings | +| `analysis/engine.py` | Orchestration, config audit, USE summary, time series, data quality | +| `analysis/findings.py` | `Finding`, `Severity`, tunable `Thresholds` | +| `compare.py` | Multi-run A/B comparison with a permutation test | +| `report.py` | Markdown rendering | +| `synth.py` | Synthetic two-socket recordings for tests and `kptk demo` | +| `cli.py` | `inspect`, `record`, `profile`, `analyze`, `compare`, `parse`, `demo` | +| `benchmarks/c/` | Microbenchmarks that reproduce each phenomenon on real hardware | + +## Failure handling + +Kernels differ (CONFIG_SCHEDSTATS, PSI, THP, perf permissions, VMs without a PMU), and +processes exit mid-recording. Every reader returns `None` instead of raising, every analyzer +skips what it can't compute, and the report's **data quality** section lists what was missing +and why. A partial report says so explicitly instead of failing silently. diff --git a/docs/design-decisions.md b/docs/design-decisions.md new file mode 100644 index 0000000..89b2c74 --- /dev/null +++ b/docs/design-decisions.md @@ -0,0 +1,62 @@ +# Design decisions + +## ADR-1: Capture raw text, parse later +**Decision.** The recorder copies /proc and /sys files verbatim; all parsing happens at analysis time. +**Why.** It keeps per-sample overhead minimal, makes recordings portable (capture on prod, +analyse on a laptop), lets old recordings benefit from parser fixes, and turns analysis into +a pure, testable function. **Cost**: recordings are larger than parsed numbers (~1 KB/s gzipped +for system-wide sampling), which is acceptable for minutes-long captures. + +## ADR-2: Zero runtime dependencies +**Why.** The tool runs on production hosts where installing packages is a change-management +event. Python 3.10+ stdlib covers file I/O, ctypes (for `perf_event_open`), JSON, gzip and +statistics. YAML configs from v0.1 became JSON for the same reason. + +## ADR-3: `perf_event_open` via ctypes instead of shelling out to `perf` +**Why.** `perf` isn't installed on many hosts, its version must match the kernel, and parsing +its output is brittle. The syscall ABI is stable: the 112-byte `PERF_ATTR_SIZE_VER5` struct is +accepted by every kernel since 4.1. **Trade-off**: no sampling, call graphs or symbolisation. +This is a counting tool. Sampling profilers (perf record, eBPF) complement it. + +## ADR-4: Attach counters before exec (`enable_on_exec`), not after spawn +**Why.** Attaching after the child starts misses its startup and races with short commands. +The fork → block on pipe → attach → release → exec handshake (the one `perf stat` uses) +counts exactly the target from its first instruction. `subprocess.Popen` can't express it, +because it waits for exec to complete before returning, which deadlocks the handshake. +Hence raw `os.fork` / `os.execvp`. + +## ADR-5: Ungrouped counters with `inherit` for commands +**Why.** Counting all threads and children needs `inherit=1`, and the kernel doesn't support +`PERF_FORMAT_GROUP` reads of inherited events. Each counter is therefore read separately and +scaled by its own `time_enabled/time_running`. The report flags multiplexing so that +cross-counter ratios (IPC, miss rates) are read as estimates when it occurred. + +## ADR-6: Saturation over utilisation +**Why.** Utilisation is the most-reported and least-diagnostic metric. Findings trigger on +saturation signals (run-queue delay, PSI, direct reclaim, compaction stalls) that directly +represent waiting. Utilisation is kept as context in the USE table. + +## ADR-7: Correct NUMA semantics, and mismatch instead of "remote %" +**Why.** `numa_miss` is a policy counter, not a locality counter. Locality comes from +`local_node`/`other_node` (allocation) and AutoNUMA hinting faults (access). For a process, a +naive "remote %" under uniform access flags any process spread across nodes at ~50%, even a +perfectly interleaved one. The finding triggers on the **mismatch** (total variation distance) +between where threads ran and where memory lives. A unit test pins that distinction. + +## ADR-8: Rate-limit page-table-walking reads +**Why.** `numa_maps` and `smaps_rollup` walk the target's page tables while holding its mmap +lock. For a 100 GB process that takes milliseconds, and it contends with the target's own page +faults. They're read every N samples (`--slow-every`), and the latest value is used. + +## ADR-9: Synthetic recordings as the test oracle +**Why.** Most findings need hardware a CI runner doesn't have (two sockets, a PMU, memory +pressure). `synth.py` generates internally consistent /proc text for a two-socket server +under named scenarios. Tests assert that the *exact* set of findings fires, and that metrics +recover the injected ground truth. Real recordings in `samples/` keep the parsers honest +against live kernel output. + +## ADR-10: Noise-aware comparison +**Why.** Performance changes of a few percent are often within run-to-run noise. `compare` +uses a permutation test (exact for small n, distribution-free, stdlib-only) and requires both +significance and a minimum effect size. With n=1 per side it explicitly refuses to call a +result significant. diff --git a/docs/examples.md b/docs/examples.md index e69de29..00b2dba 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -0,0 +1,82 @@ +# Workflows + +## 1. "Is this host healthy?" (30 seconds, unprivileged) + +```bash +kptk inspect # topology, caches, THP/hugetlb, risky sysctls +kptk record -d 30 -i 1 -o host.jsonl.gz +kptk analyze host.jsonl.gz +``` + +## 2. Profile one command end to end + +```bash +kptk profile -i 0.25 -o run.jsonl.gz -- ./my_service --bench +``` + +Hardware counters attach at `exec`, so the profiler itself isn't counted. If +`perf_event_paranoid` blocks them, the report says so and still covers scheduling, +memory, NUMA and huge pages from /proc. + +No counter access, but someone can run `perf stat` for you? Pass the output in: + +```bash +perf stat -x';' -e cycles,instructions,cache-references,cache-misses,dTLB-loads,dTLB-load-misses -- ./my_service 2> perf.txt +kptk analyze run.jsonl.gz --perf-stat perf.txt +``` + +## 3. Watch a running service + +```bash +kptk record -p $(pidof postgres | cut -d' ' -f1) -d 120 -i 1 -o pg.jsonl.gz +kptk analyze pg.jsonl.gz --md pg.md --json pg.json +``` + +## 4. Prove a tuning change (A/B with significance) + +```bash +for i in 1 2 3 4 5; do kptk profile -i 0.25 --json before_$i.json --md /dev/null -- ./bench; done +# apply change: numactl binding, THP mode, thread-pool size, ... +for i in 1 2 3 4 5; do kptk profile -i 0.25 --json after_$i.json --md /dev/null -- ./bench; done +kptk compare --before before_*.json --after after_*.json --fail-on-regression +``` + +Each metric gets a two-sided permutation test. "regressed" requires both p < 0.05 and a +change of at least 5% (`--min-change`). + +## 5. CI performance gate + +```bash +kptk profile --fail-on warning --md perf-report.md -- ./integration_bench +# exit status 3 if any warning/critical finding: fail the pipeline, attach perf-report.md +``` + +## 6. Case study (synthetic): the misplaced database + +`kptk demo numa_misplaced` (report: [`samples/numa_misplaced_report.md`](../samples/numa_misplaced_report.md)): + +1. **Critical NUMA misplacement**: threads ran ~100% on node 0, while 85% of the heap sits on + node 1 (placement mismatch 0.85, expected SLIT distance 19.4 vs 10 local). +2. That explains **memory-bound** counters: IPC 0.60 and LLC MPKI 17.9. +3. Node 1 is **nearly full**, so new allocations go remote too (35% remote allocations; AutoNUMA + sees only 25% local accesses). +4. **THP always/always** with a 44% fallback rate and 12 compaction stalls/s: the heap is only + 4% huge-page backed, while dTLB misses are 3%. +5. And 4 GiB of **hugetlb pool** is reserved but unused: memory that could have relieved node 1. + +The fix order follows from the evidence: bind the database to node 1 (or migrate its pages +to node 0), switch THP defrag to `defer+madvise`, then either give the 4 GiB hugetlb pool to +the DB (as huge-page shared buffers) or release it. + +## 7. Reproducing phenomena with the microbenchmarks + +```bash +make bench-build +./build/mem_latency > lat_4k.csv; ./build/mem_latency --huge > lat_2m.csv # cache + TLB cliffs +numactl --cpunodebind=0 --membind=1 ./build/mem_latency > lat_remote.csv # NUMA penalty +./build/page_fault_cost 1024 # fault cost by mode +./build/wakeup_latency 200000 0 1; ./build/wakeup_latency 200000 0 +./build/false_sharing 8 +``` + +These are deliberately heavy. Run them on an otherwise idle machine, and never in CI. diff --git a/docs/linux-performance.md b/docs/linux-performance.md index e69de29..4116bc5 100644 --- a/docs/linux-performance.md +++ b/docs/linux-performance.md @@ -0,0 +1,81 @@ +# Linux performance: the kernel mechanics behind each finding + +Background for each domain the toolkit analyses: what the kernel does, how it shows up in the +counters, and which microbenchmark in `benchmarks/c/` reproduces it. + +## Method: USE first, then drill down + +For every resource, check **U**tilisation, **S**aturation and **E**rrors (Brendan Gregg's USE +method). Utilisation alone misleads: a CPU at 70% average can have a run queue ten tasks deep +in bursts. `kptk` reports a USE row per resource, then domain findings. + +## CPU scheduling (CFS / EEVDF) + +* Every runnable task waits on a per-CPU run queue. The time between becoming runnable and + actually running is **scheduler latency**, counted per CPU in `/proc/schedstat` (`run_delay`) + and per task in `/proc//schedstat`. It's invisible in `top`. +* **Voluntary** switches (blocking) vs **involuntary** switches (preemption) tell lock and I/O + waits apart from CPU oversubscription. +* **Migrations** across cores cost cache warmth; across sockets they also cost memory locality. + The load balancer trades these against idle CPUs. +* Mitigations: right-size thread pools, `taskset`/cpusets, `isolcpus`/`nohz_full` for dedicated + cores, IRQ affinity, SCHED_FIFO for the few threads that truly need it. +* Reproduce: `wakeup_latency` (block/wake latency percentiles, pinned to the same core, the same + socket or across sockets). + +## Memory and page faults + +* Anonymous memory is allocated lazily: `mmap`/`malloc` reserve address space, and the first + touch of each page takes a **minor fault** that zero-fills it. File-backed pages not in the + page cache take **major faults**. +* Allocators return memory to the OS (`munmap`, `MADV_DONTNEED`), so steady-state churn can + keep faulting. Mitigations are buffer reuse, `MAP_POPULATE`, and huge pages (one fault per 2 MiB). +* Under pressure, `kswapd` reclaims in the background. When it falls behind, allocating tasks + enter **direct reclaim** (`allocstall_*`), and PSI `memory full` shows lost throughput. +* Reproduce: `page_fault_cost` (base vs `MAP_POPULATE` vs THP vs pre-faulted). + +## NUMA + +* Each socket (or sub-NUMA cluster) owns memory. Remote access costs ~1.3-2x local latency + and consumes interconnect (UPI/Infinity Fabric) bandwidth. +* Default policy is **first touch**: a page lands on the node of the CPU that first writes it. + The classic bug is a single thread initialising a big buffer that many threads on other + nodes then use. +* AutoNUMA (`kernel.numa_balancing`) samples accesses with hinting faults and migrates pages + or tasks. It helps long-running unpinned workloads and hurts when threads bounce around. +* Policies: `numactl --cpunodebind --membind`, `--interleave` for shared read-mostly data, + cpuset `mems`, `mbind`/`set_mempolicy` in code, and `migratepages` after the fact. +* Reproduce: `numactl --cpunodebind=0 --membind=1 ./mem_latency` vs `--membind=0`. + +## Caches + +* Typical server hierarchy: L1d 32-48 KiB and L2 1-2 MiB per core, L3 tens of MiB per socket, + then DRAM. Latency roughly 1 ns, 4 ns, 15-40 ns, then 80-120 ns. +* Dependent loads (pointer chasing) expose full latency. Independent streams let the + prefetcher and memory-level parallelism hide it. +* **False sharing**: independent variables on one 64-byte line ping-pong between cores through + coherence (MESI) traffic. +* Reproduce: `mem_latency` (latency cliffs per level) and `false_sharing` (packed vs padded). + +## TLB and huge pages + +* The TLB caches virtual→physical translations. With a few thousand entries of 4 KiB, reach is + only a few MiB. Beyond that, every miss walks a 4-level (or 5-level) page table. +* A 2 MiB huge page covers 512x more per entry. Two ways to get them: + * **THP**: automatic. `enabled=always|madvise|never` decides who gets them, and + `defrag` decides whether a fault may *stall* to compact memory (`always`) or defers to + khugepaged (`defer`, `defer+madvise`). `always/always` is the classic latency-spike setting. + * **hugetlbfs**: an explicit reserved pool, predictable, but the memory is set aside even when unused. +* Fragmentation makes huge-page allocation fail (`thp_fault_fallback`), and compaction to fix + it costs CPU and latency (`compact_stall`). +* Reproduce: `mem_latency` vs `mem_latency --huge` (TLB-reach cliff moves out). + +## Hardware counters and their limits + +* `perf_event_open` programs the PMU. A core has only a few programmable counters. Beyond + that the kernel **multiplexes** and scales by `time_enabled / time_running`. Ratios are exact + only within an event group. +* Generic events (`cache-misses`) map to model-specific events, so compare values only on the + same CPU model. +* Access is gated by `kernel.perf_event_paranoid` (Ubuntu defaults to 4 = no unprivileged + access) or `CAP_PERFMON`. diff --git a/docs/metrics.md b/docs/metrics.md index e69de29..61e5500 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -0,0 +1,66 @@ +# Metric reference + +Each metric: where it comes from, how it's computed over the window `[first, last]` sample, +and how to read it. `Δx` is the change in a cumulative counter, `Δt` the elapsed time. + +## CPU and scheduler + +| Metric | Source | Formula | Reading | +|---|---|---|---| +| `utilization` | `/proc/stat` cpu line | Δbusy / Δtotal (busy excludes idle, iowait, and guest, which is already in user) | How busy, not how *contended* | +| `steal_frac` | `/proc/stat` | Δsteal / Δtotal | vCPU runnable but the hypervisor ran someone else | +| `iowait_frac` | `/proc/stat` | Δiowait / Δtotal | Idle with I/O outstanding. It's idle time, not CPU work | +| `runq_waiting_tasks_avg` | `/proc/schedstat` field 8 (`run_delay`) | ΣΔrun_delay_ns / 1e9 / Δt | Average number of tasks runnable but waiting: direct **saturation** | +| `runq_wait_per_timeslice_ms` | `/proc/schedstat` fields 8, 9 | ΣΔrun_delay / ΣΔpcount | Mean scheduler latency per time slice | +| `psi_cpu_some` | `/proc/pressure/cpu` `some total=` | Δtotal_us / 1e6 / Δt | Share of wall time at least one task waited for CPU | +| `process_runq_wait_share` | `/proc//schedstat` | Δwait / (Δwait + Δon_cpu) | Share of the process's runnable time spent waiting: its scheduler latency tax | +| `process_involuntary_switches_per_s` | `/proc//status` `nonvoluntary_ctxt_switches` | Δ / Δt | Preempted while runnable, so **CPU contention** | +| `process_voluntary_switches_per_s` | `/proc//status` `voluntary_ctxt_switches` | Δ / Δt | Blocked (I/O, futex, sleep), so **blocking/lock contention** | +| `process_node_residency` | `/proc//stat` field 39 (`processor`) per sample | histogram of CPU → node | Where the threads actually ran | + +## Memory and page faults + +| Metric | Source | Reading | +|---|---|---| +| `faults_per_s`, `major_faults_per_s` | `/proc/vmstat` `pgfault`, `pgmajfault` | Minor faults cost ~0.1-1 µs; major faults block on storage (~10 µs NVMe to ~10 ms HDD) | +| `process_minor/major_faults_per_s` | `/proc//stat` fields 10, 12 | Per-process fault rates | +| `direct_reclaim_stalls_per_s` | `/proc/vmstat` `allocstall_*` | An allocating task had to reclaim memory itself: latency spikes | +| `pgscan_direct_per_s` vs `pgscan_kswapd_per_s` | `/proc/vmstat` | Reclaim by allocators vs by the background daemon | +| `swap_pages_per_s` | `/proc/vmstat` `pswpin + pswpout` | Any sustained value means the working set doesn't fit | +| `psi_memory_some` / `psi_memory_full` | `/proc/pressure/memory` | *full* = no task could progress because of memory: lost throughput | +| `mem_available_min_frac` | `/proc/meminfo` `MemAvailable / MemTotal` | Minimum over the window | + +## NUMA + +| Metric | Source | Reading | +|---|---|---| +| `remote_alloc_frac` | node `numastat` `other_node / (local_node + other_node)` | Allocations placed on a node other than the allocating CPU's. **Locality** | +| `policy_miss_frac` | node `numastat` `numa_miss / (numa_hit + numa_miss)` | Allocation-*policy* misses (preferred node full). **Not** locality | +| `hint_fault_local_frac` | `/proc/vmstat` `numa_hint_faults_local / numa_hint_faults` | AutoNUMA's sampled *access* locality (needs `numa_balancing=1`) | +| `pages_migrated_per_s` | `/proc/vmstat` `numa_pages_migrated` | AutoNUMA migration cost | +| `process_memory_by_node` | `/proc//numa_maps` `N=` | Where the process's pages live (huge pages normalised to 4 KiB units) | +| `process_placement_mismatch` | residency vs memory_by_node | Total variation distance ½Σ\|cpu_n − mem_n\|: 0 = matched, 1 = fully apart | +| `process_remote_access_frac_est` | same | Σ cpu_c · mem_m over c ≠ m: remote share *if* every thread touches all memory uniformly | +| `process_expected_distance` | + SLIT `distance` | Σ cpu_c · mem_m · dist(c, m): 10 = all local | +| `node_free_frac` | node `meminfo` | A full node pushes allocations remote (with `zone_reclaim_mode=0`) | + +## Cache, branch, TLB (hardware counters) + +| Metric | Formula | Reading | +|---|---|---| +| `ipc` | instructions / cycles | < 1 on a modern core usually means stalls; > 2 is pipeline-efficient | +| `llc_mpki` | cache-misses / (instructions / 1000) | ~80 ns per DRAM miss: 10 MPKI ≈ 2.4 cycles/instr of exposure at 3 GHz | +| `llc_miss_ratio` | cache-misses / cache-references | Generic events; mapping is CPU-specific (LLC on most Intel) | +| `branch_miss_ratio`, `branch_mpki` | branch-misses / branches | ~15-20 cycles per mispredict | +| `dtlb_miss_ratio`, `dtlb_mpki` | dTLB-load-misses / dTLB-loads | Page-walk pressure, which huge pages reduce | +| `min_running_frac` | time_running / time_enabled | < 1 means counters were multiplexed and values are extrapolated | + +## Huge pages + +| Metric | Source | Reading | +|---|---|---| +| `thp_enabled`, `thp_defrag` | `/sys/kernel/mm/transparent_hugepage/*` | `always`+`always` = synchronous compaction in the fault path | +| `thp_fallback_frac` | vmstat `thp_fault_fallback / (alloc + fallback)` | Huge-page faults that got 4 KiB pages: fragmentation | +| `compact_stall_per_s` | vmstat `compact_stall` | Tasks that compacted memory synchronously | +| `process_thp_coverage` | smaps_rollup `AnonHugePages / Anonymous` | Share of the heap backed by THP | +| `hugetlb_pools` | `/sys/kernel/mm/hugepages/hugepages-*/` | `free - resv` pages are reserved but unused memory | diff --git a/images/architecture.png b/images/architecture.png deleted file mode 100644 index e69de29..0000000 diff --git a/images/cache.png b/images/cache.png deleted file mode 100644 index e69de29..0000000 diff --git a/images/numa.png b/images/numa.png deleted file mode 100644 index e69de29..0000000 diff --git a/images/sample_dashboard.png b/images/sample_dashboard.png deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d1925eb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "kernel-performance-toolkit" +version = "0.2.0" +description = "Linux kernel performance analysis: CPU scheduling, memory, NUMA locality, cache efficiency, page faults and huge pages" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "Apache-2.0" } +authors = [{ name = "nareshns2004" }] +keywords = ["linux", "kernel", "performance", "numa", "perf", "hugepages", "scheduler", "observability"] +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: POSIX :: Linux", + "License :: OSI Approved :: Apache Software License", + "Topic :: System :: Monitoring", + "Topic :: System :: Operating System Kernels :: Linux", +] +dependencies = [] # standard library only, by design + +[project.optional-dependencies] +dev = ["pytest>=7.0", "ruff>=0.5", "mypy>=1.10"] + +[project.scripts] +kptk = "kptk.cli:main" + +[project.urls] +Repository = "https://github.com/nareshns2004/kernel-performance-toolkit" + +[tool.setuptools] +package-dir = { "" = "src" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] + +[tool.ruff] +line-length = 160 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"] +ignore = ["E501", "RUF001", "RUF002", "RUF003"] + +[tool.mypy] +python_version = "3.10" +files = ["src"] +disallow_untyped_defs = true +warn_unused_ignores = true diff --git a/requirements.txt b/requirements.txt index 55c084d..ecb4476 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -# The core toolkit uses only Python's standard library. -# pytest>=7.0 # required only for running tests +# kptk has no runtime dependencies: Python >= 3.10 standard library only. +# Development tools: +# pip install -e ".[dev]" diff --git a/samples/laptop_profile.jsonl.gz b/samples/laptop_profile.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..2597c50c4621efa4aeaeac07ff3d031d5e042e61 GIT binary patch literal 12345 zcmZvC1yqz@x3}U53?VU+Dj+d*Gaw}bGEzfIcL>7(lF~VJcMaVg;sDa2bTuGYEBkLZ${EbOpS;_QFU9Bs%&T5vFHva5;KEhRF zOwVJCw{5T?h_464-W%!Mz4zmK$H;QGS;p~#LnIx%d$nifT%R?{e04i)dJCqf*h3oc z?R(Yr%*}gc>JM+(@f^TvGnfu^KbUa!32$uv2B$F%9up_5Zyb?>mY+aWu*@`f-5Qmk z#By~^P`R$5pvU3(oOJGDdd!0j$L2Li_8jw2-N<7ExGo8sn`)$Z(k7t1)_J-RdM{`}VZmXsI&6tugX2YfR!#TwHbD?lm9v>vbJ;q@_?lh^=4i z?kb@9k10ehz8iEiJug9*Kg_1{=)HuoIIvUqt5Cl&{Calu16(gG>K%-B?s(uF4xQ~MK;?Xi+Y?U#DjDUY zWjnf*bGKSYU0MDsHso$j&fUZ5W1&f>?91|OBPh4;?Ll+4(I3I9i|Tf-+fyR$yu+0B z?cHJ6=d(xQVfE};IDRM#z zQwGx*>S~-xa}lHO`%U&DS%g(7a(r$9-fWyY(lKW#jB1zuLWv_MLnays63X z@+uFqdk%5%Rc4K`36b-%-?$ilH&a(^1x)j}NaNv{lxOe+_&?bV*m2{#|B%qYy6XjV zLF_tW0(0?)o=8+ESG_gvs&1RYz?{eJiMsoF~ zi{b*s!Fn(HQuU-i`J^=YWGDHA?37)$yd1TOxR>Pg zmiy~nYu68cEQns;f7V=?fv)>6RMchs3oUgS|3Y3}#=lTom+>!@)n)t(O?4UnLee?E z=;n=}VQ+6$&U~xHkml}&#b=sP%U-TR9rE_kp4oMC=RFUPy?u1R5xV&Zo&FaU0Y??h zM-><){ir42sHOR+1*4=N(U7Yl=bfgzjw%D)Z%MN{8jEDZhyhHUQBGw`)rf5%7t zs!R3F)pX(K5Pfa##-nfgBj@A0XSe!~+`*EUKR34jvgjH7h4K_>n%bIH4`C6C83nG& z^DfN8*kY(xwPEx~(0ROqb!({MCdK&RZ+ee3qu1OXajiq#wJFy!7kuq~hG%&g@-6pi z#Lss=s;%2HmwfHpn9)mvzwM(h&a@jdxYctlpUS2@c60S%l~hgMWax`jl- zjYRUicS0`;8NQ`NYa5A}VvO@bhMAOTHzScXjB#1W@I58^wUG#^_m1pYA;Wk|w7HQ; zJjTGxnNNv+Yb2tEF)(wcQ=*-XL?$o>X3kPdH13~9L-5_#l&xb(Pbc9GY-m$q$e7nR zg+uFj;YVyxeTtX1^7T>H35oaygXh4T^Te0~RLp5$heQ?5GxKsCx^awz3nXu>+@_`T;Trs$xjeQ3h_rw3~FV?mS3(BQ6s@Wki_(l zdAstmbBi<`^Ru%IVwh8%HYY=rd}2ABH(5;ML7gw6049mhaNL2^(A14YYLnNDoDGq$ z8P^E7)Jc2-@zw(IRCTRo7+CAV&}#7mq3rzyH3Acns)Q8M;vDRYP=-EiX~wYlIKgl} z{03eNvX60_0LSNXdHQKxwgmF|iz^f``<_Va|*-L{24g_`LVfi+?z$wOe zEa`<$?LbqyDsg$i>$Z{9s3pOy$`rSB^jK8M2R@!*?|OIR33#sht$N?H!{RaJ0`wKh zLCInbHyn>%ZQK0K_e<7*#{zttpFKIwtmG77;aarj5u8-2T>=p~map)X*hpN2;QT!d zW@*9IQm@Adoy{e;?5$r5`;QNer=)=T=?8M?K8JSsl0%>WCga(rGi0G;IeNy2+*>g! ziI24RQeP7x=qyk2n_H1CGxE7}(o&<*s#qH`Ra*{7xaz4# zaIbbhBB|o3S_D6B9W{s0r+GIQ&_|9n1Rrgi95nS1;nPaPeHZr2B%hEO&*gif72CtT zS=KltZ?|m$V-)2VbwOadqis_6QcI27_X2vEq@MlH(Is#7SN#(h^>sH4efuBH1vm8f zW!JuI;b1cwRe2Pe#YLYR#+wY!{jI9Yi(5o#b1qDg!3J^|2_q)heelbrhcn<=+sA6l z{A&T4QI+sNiP@WS`MbL`PaYH9%44Z@=S+uACn~7q(&6c8;YuBKMDfa33h3RZ@;t+; zE%T#`sL^%6G6-@K!x4B|Ff)g&+OU;;^_-^FuZ5n@S;^{XQI&W95tNBQX|QpI1QbuX zU5H@M;f*oua0KCECB%G0seg$?V)Ie|sZ-OTv_0yFQx+COu;0~1uwf^J=RMw3-p+g5 zk(wxlFv*GNvS-W*NT3cVqo*6%BUSE_d@a;(pK2Samdvu9M^L2kWlAne(tw2c=DFq0 zaW&w6o#tz*gpoYDmmA+|C#eFiPp(~l&W_CU6_PbEn~}Vv?0QFT`i#m!EFxt0c)^@_ z)Y!W&xC{LW##z*fkbm~+@O$NzIYB&C???T{>B8I(bn}9?{9l+`tpn+r-c_h%yAB>x z|3nQz;r{k$eLAafY+S|IgTZMp{wLtL>P@yJ_S;RsGbOXT4-X(u!4V%2O)s!E_9=(x zwHc3hh{s<#iWpX&uvcvjntSmNwu=()ForFINvZ_79+iQV^$GPy79zZ zi)r`%%Tyy7O$8JAC*tA~2|#O_HJmBZUV+xBZTd zPtstMl7%s??3|3-_)Np#wTHO>_hhEx;~n;UFQ6fX(ffsI%HV)*H|*B6?DS(&`8=sa z^<-e=ca7)WTeznTa9Y9p9JVAl7a!L98!`oqO^SZPf-e)iBU&OT4mQeE&6zH~bQ2D> zPYT0L>!{YS@wpeF$Xe(+2@6{k4HMS|p#?vijQ?i*YLl172p1>(@)0DZ;{6sVHkQ{6 zuhrJ?*-r}~XtTtgtc@(kAoG0>znA-ca>*`tYX0R4hD9CRIc6z*Zq372A#nv*n$c(M zhW_Fm#zfkSUqPvaIAfVB^qS!-9BWgE2s>sQkj3f7W3aL$(B|f`V#Gqh&Trcpw7cLz zCS9b%a_(@B#+w}iqiFw$0f#m^3o7nz%R&{x!B@g5Ej8W4KZo7o$k4agK`N*>SEMCi zwix#Q*zcNsWE3T`vU92H{0DE@eMAC+{g|+}L~IVlW|joQBo-U>CiNBx@X{?(6$C0k zRh-4gcMAEPN}%MOHQP3`&MDC>#l`vnx$=pUVLX9O6aM$9F2fDF%*TwovASLL3Qs=5 zj+Wxr9;rWG!}m^T$`5=Z#C_$}k%Q$>Ikf)CrXUv;x4&`TSfI6~{u$ z_>+8I>=?jq_MRQdA=589{`zoKzoS-vgyMa&BfolLw{hX|px4W1EuB@d^Dm6)2?g;p zlImTE>tB0gX}^9jJBwIvaY(e>w;!H79_BN&EIV610g^`A0QCs+Bf?64;VxIJQd5kb z?{{tS1s9g_)Fp!!4K7W!BKqfGgvy&dcN z`kuRQh=DRJuycl7)Fyco{1{%_wNZtLq~JbLTPYb?gYpiY*NpYp8AI)hq?rn+} z-m#}>GOZZP3tOMNh&NK}+d}i3B}InG&)m5=)vY>Lo_4HI5eR{vJ`I;zoq4C%O3$v# z`@n@xMzq=kSRwZXhQjZBkcm<#`g0a{ijr3p;?Ev3hY<$XoN-UScdd4WA4KvPCAm=8 zHRKKP#JlmHP@85}7AYE>8M{5onS!?iu2y=dkMbAq*_)9R=^I3`4>_f-tQ>EPYkiIgn}AJ@3u0R?{*+UG1( za{>{GwC)F#PlI%NasY$EgYC`xOawBVEF+tf{-3gPb|_qT_D&y>zqX|ImEC8%>z^f+ zXmjF68y)44;6yw2sYP|X4YPKDBw3cu91>{l4q^9?J}~yZr#3R0m3f(m1~Z23z?T%pY8pn%rGKGJp{eCnQ|B{E}CHAJYwZwn{FC5yUNeyn`j zdRMOFpii;*R*EcUvL0LPDux?aHBd#}&Dr;tdY?f=glXnn$2jelBMCv9@m3n2Pk~i8 zMveFcK|qbriA;CC1|I(_ylYX1r-iO-AbhCe^@!L!v3AHLpYO5~-zeWcn_SDyL|R!% zF0K$$BdPrGn$m@Lxv^=ZPKSk=fJWDH&1n*U_g$Z$gDU3~vrmL)pgbPsjaB_!rmbxD z^x`!HzRy&(mzFpdl#}8nT+XDH*Qt5txTaK>%EUB^ZT9UdCzF(a#1!gqW(ub&VncBRjoKZbf$FJh zwv-Nk4^F}2+-bj95|L9pE{xb|udat#Qk}2QGPiwR8v4FiJ~?PEtZySfq0-grarcz_ z&7rZ2FRu}oklW+SiixN{1TGdI**C(fiWAiu3#Tl1NsW}=B()`fzZQ=i%A7Y{yM5xl zc6c4O`so*8f~7DE;afFD)~E@!f$H*x{F_9r^o=~!+R)~4h`H#KtI~U*PAO0Feaf*K z9zQs}H4_gZSt(;}_GFbbrxJZxhU{J9ch)rH9_9UOdcL&Y{HuLJ4*D~!iS36VlExZA z)06|-pJxqe=Bv>UDc#D4`6R1p&$?2T6L1}Keh4ozALn<)sI9(HTC@aJs(?KX zYRHqtD9d|)c53GfBz(5)LxXkbZj>{;O^&IWm; zmMb(1njl?PSEuLWaQQlFbMOMSq&akfTGPkD&t{TBO+ccO!`h3+Umed1d}A&2a!F-R z@;zX zYF+I3)G$L_0x(0o`UVha`Uz|5D4;JaAEr9(lxY>$KW#4e?!$Y&?po_d{l)z}?C9B% z?v^>jX2qa2NHa&U6zzhRX8vt#OeQvemyxyYxeZ&cP5 zt^9Aco9XJA#XE&b-^})J!cl-aH$OaaqtY^?^wY0hckH$Vku(pQN%Bn!I&rkQ9TM0c zJL6y}kbhKDbME^@9aC7ySR?B{zS@q+2Y!5N0iZ0!n|T2o1TVLN{bg2;ofBnLpOG!+ z$$e4j#z$vxSV{lcyb&;A-9FZVH zx462Iu;|w+`&zqYbk*dS^x-Kgkvg^v0(1BhMeQ+(@@FYKgN)Gq#>}d-@*Sh1wyLDL z?BG8W`*Z;WJ)eHX`TjY~_GWY*-n%d|YAu8QY0_P7SK-#X`uXfaOLq7ic z<0vTq%Xdd&u{p{G9A(#*)-s=T_61YzzFxYUD4-D9{pHFl=BE3br@3;NINp$}xH-P9 zx$S$>*>L>{x`asf#=}FybC&zmDBhJjWxb*N?R?Glx{mh_e%3)_f-{3?U-HZg z6s=d^6|LLaXLf{tat78NKd?(QJGtE`JJN(uw$^Y?l9&zDgwE<`9$c?Itg@T`yF9n9j>7`E%fzc;sGz z+0kbDR*l+~DS>DaCD`4YH+4*fU~BWU>0FhmwmPp`FAiBibJLcHWA*!v_HI&Vvp>N! zS?z0W_pVepBq0d-M{*x=Q`D1Hg|mZg&gx?GO$8g|T4YbVU}mT%K5XZY7d59gN?3z`WD$J6i5DL(T$x_v|$l~e@a#-9`Ekz+MP6BhDK z7QR(-{?1v-DTF-^d$>@p%Iq*)zZl+9R66H2%Q&IGci}Q!HdAG1zEq*My!o?df6%P0 zWgu`||ND_)O7hDmIs;Hai~I!7`HrIA2<5?($)b!ZS(s;~#qpYBD9#7PQvGKKQ{0V_ z>zxgE?jLoG(g(8*2@HQ<%%&{cdRl+Ck}a$joVjoGu~c?aszK}I=jZz0eztPla}Nuz z`C@@|{k#ujL?{$BG;3K>B15hSWy%XajJwn5p2ddPR3n{}`es6F&!W1l&sp)Nd&y8O zMqhW)#!k7X*AVOXGLNz=tIjlN-`V{_i4>L3D+O)Mf33~7dCRPU&N4ThY+V#w;;QdW z66NIFa$Z<=?gm(MoXfkiYHUri(#$2>e1|d-3EQdL_%zGuPB+;u5DcI9a*ooNp%)2E z?$Z8< zI)A=o8ZLVxN*X0>eRLA~>Qr^sa&FvGfOl?BK}%E~Xom630*dU9j$J2TJ(Z0c>>f7K zy14Pmt<7IFmh*OhMP&ZAmLHt5t#G#fvFzJV8qu&>cS~jz3bXjKw%(#I&9c7a+@>?l zy4tHHAyKaCqnn$(M&z2*a1-qgDcjjE&Lm=K4KA}INrJqUTWTx3wn<@0)t=O7$$)SJ ze}v&k>ejvf##}9>%Sje`Z!9@~Nt(z6{f$09NqrQqu}2JLZytpwCGNdL|D=YpPkK^S zwgiudUrmZfYN{Kfbw}Yc=v^9UIR}q!k|mnDF=d?1nzAyq(zX+>;mn<{K|xu0AlQiu z!OWAw{UDI9AzWpx0A)87ZUaDf#XrPX-V8AqqY>NY{^QjD^L7YxS=5zpf|d`klWQ_>r0$-}l9{#t}~_D>daMSq7;pRiyJ$ zEzxY{%30%VB{e-PyHu5Q$zP4$J4R|kg+<+=8uWlW}7=0obsR^SA9cN=%#U1-O1W$q=J?(-IKWeBQ@x`RlklYbUP8;K9=TrPQdUZ_ zUKjG|N*)&R=TEE6O<#1wA#6Ol5Aq@ZE~dvzjX&*`s$Ei5>dzDSPb0^FN*cW3(~X*@ zdS4ms4Xz*-rL3&6W*}>_4s%IX{rr>HGI8GmZLwE7>OYQSa7+{>S)P4uOQ zT6c!ecf-4T2U~_TZF}LIlq`giGmA>|{cz4yc39*Lo+x_U&B_$5c~wp{2!|BVLalWq zqLO=+Ff*O4%+Zc@+2gv&t>a#}kux3e+rY>ftq_c>=Ym`LP*dgtjR4mBy9K9Mfz!WD zcq-4FI?!$5SruTpTHK780O88FWIhP`Ep)PJ%obYhA%9bey)J&7j1s#@lm~y)kW~`& z*^pJ?gvG95c7|q_-4RwX#9sG7@wtF-r43QBhB$Gt#;Wx5=SvR^QU`|orvU6!kk@ZZ zfEiDsIfS^DzC3+S)B(m-?E>?}z{XItey|nD9H0hzVFFSl6udw4*?SJ4+5wi0fz?!y z2}&OJf=hnQ0i>@CZIDeNuzL!<=q|8vp`J15^Y;kMeAN0!pj1OI_!ZO$ASij9IS1HY z-LC=Oyt2WPxR-8xSPqp0VEeAeiui<{JND_ z?)Ytx=JHm_rXYZDA@66}bU&O0+mFDE+L%a~F;NJNC;1$SNsVhK*j$8gE)etu>?b*o zm)_B)Z-Z4_91K%o4u-utJiMI)#CdG_de|U~biGP35x%9J0|@!tbb?d;V7POE{~@MU z$EXi8Gy`BAAhDD2kH9WZg{LtvW5W?Y*jyBh2a|LWr_MjWKLY8EL71EygX-r1GuY`l z=j7>hE~cQPx;;PGRl!`~E%OGB320IBn)wU(ho<5KLH{{E-CT&^bAFX3Q;@b!)hKSd zF%3xYxsbujM}m?78>tTP|4(F)3U(T*^sl5>7=t*3$U_*KtoY{?pS^;`qF=*QvQWmQ z;_;{)F^J@L>lC255Ny9z3fvZI3qj=^sDobhVMsk`m|u_+Eku%{1o%xI)Im4@2F3%K z0u)8(fn}9pH7?wc&M6g8!8Zk1jr`ntDNxNN2P`|H+!v4!fpkt}P61eH9zZ$`o|ORq z^N5K(LSQl1^xDe6ay(0c@g)1DKopNM%rD9j1i3Fb9gXs{^GqoPBKriV0N-E1a^5OL zTk=prI&+qDi@|ewqqkvdAc0SSL^~%0QbUbZ8MY!5vN>4*R_O{uf(J{0W6D7*i|U}o z-3?|XScfzWI*~8`@WB8)`tY(~3SiRTuMR4xOH|8KLm-1paF_WJNSWCcP%&6mObRXK zfk2uE^LL9wI@P>SrU1RyK}5ok&W^}YG06R7zl35i-Sc^EeXtm$Q^jU95!DfHpE>G> zdc|a`0E^8KR0R=ONyF$|gdm+_Pa*!+&a^+3VE6EQoO3b9ZK-r5pahsk`T(+;uL>$) zrrOH~6RYQeQ!)F@b5#ZPv8sYz=YYjtK>W8KlmMw1Rzd2Zocc|LLa<+2r)Lh>-o#vD zDiH1e0P zrviK5z;cM*z+R1py@FjOP6a+t8X(FA7vOAUN1|f&cnPNfViVf|sMsegQBwe!3ssP# z(?-4uh!S&P`e12QQ-MQ$+p;jaj7uquoq~;!dS3uQbg`;{>WeU;{uA$ih_hF)mH1V5 z$9D*%RJQxv`>qqhqJZR(S0Ec8%2*wa460CrV_74?q114s+OHg7O_^OTa7r44Uc$$m z(J?^Lr6h>4kqGAZ%$gmJrLBbvl!(z@Y|XfQJvh%97;( ziSEJfwYvI&OP-?c6^YF4(V&cP0>RqPa{l+>rGaDZ>F-NRf+CZt1SEBdU^R&pxxiWi z*h-)W9Jz@mfW=;=fA5tBnLiNlXT1!K-Wmg>w=D*Y0j^#vKSR;s_A>KonvVhA2X6LA zgBVqri7-DJuz)u%7@uKeQpIBcX%maB;sgEGoHU3f7kl$9Cd9oTxxgBz z{g<(R{vdFNBBJDE3?TWE8;-?W1QF;A$A%-1{#?G10S-h3#&mB3UvCAj{$V>VLA6vV+;bF8nisydU2cT4wO$tj~$Z0PD{v z5aNVEm)}H5;7GgCb<8X+6MQ(5a4Zlf0GxUchKo^x!8IWPU|A&ilme6MEv_O$f&1Ae z&H>=3_G&)30?)-+?wrR0hr+FpVBB&Kf3P{WJSMsZbyN;ezdQyT#`Dhd9I9T#FOW;{ z2u_GGOI};Ur5=Wk!3zNYH|l>v{~zM&NJkP>Hd`#(?&hzMA#j{TN%GqPn>1>|)t!D( zMy!3YA~{8CeZJ7>4;M?=L zN$=wT(<$-*AYr%t`)JnS&X=|Il!|Z`=+CPhQ+vBh{Wk{>A6+$QBi|1#k9|4STBNu~ z8_v`*TSOmRpdC0UcNO_sXQTPk_^ecP`1~UHn0`3I-yv=@I9tAHLlmXLd@;Zn7JjHf zaM6=-vo|r?O)#07siSL#TdK8X?)4!|S}E+b->u%=*N1<>_{Whcq+P|`l2M2GBiV3l zk6PM}8k?oqVfs|^LyMMDV^7FW#;W;by|`HcvL}ZqIaI{b_bT1#o2#t!B#H*^n0MaI zn9no@pM2k)ohN1`BA!o;QaXrPu(1g}=^&2EUkjf?&mzdVxkB75dJ_kxS0CL=7<@tF z^(&hq{k*R6_fGdpx1F!|wZV72Q(i350W~Arrwm$G^)ML}c~fh=R&U(~j*WsxCl|}` zEK>2FTr{iORHJcOZM;mQxmX{9MVF~EV&rrzFGS4Tlk7A0sE1wSso{wj5k@5bQ-aJ;}@Uw&eYDG1QY_0^^-22>9OobPobJ%fBGgivmE)+;R-8u|@>YWo2 zlAdcV@*OQc9k+uEz7r$54SnDgeRIQ#o{!>j zZSh*S%dQ-|=i(+cLA7U@Bm()FSALK!XSmhMdAbg}QqNc51|VZ(6f-1yq;>~0P@1@J zRZF6iwNqj>G)kWG(bh&kyw!Itpjaqi!;5d8gDvF6mJa#*d|W_hB4F)oZv`B!Q^K{kitJxgu6fp!mx z3>D*FjOtn(5?_7&>11uR$#*W!zC?Xp+^e&8F|+3Mz^hy^jj&-9fKM$&by##dT&4t+ zXuG-Ai8`75`cE&a^&rN*y=;f9EK^w+(~DNq|LsMYt|KBxUBZxIm(B8E%i(lv_ch53 zEmkUq`bhWgHwobm5jas$yuwE-Eo>S;TDn@&9Sx~?S+|lbGqeo8xF)~88dFoG%7>ui}j0YdGuBb+c(-rtGrH_75d#ZS_a$;4`5uMLQdKGXhzr4fWep{5Gr4-5KWLOt( z`=YwZ|1|uKVz|?qaCOG|XtPCz7EZs2!w9=&4j@mX$9)>Q@=<2MU4ywNwPL*`&ndRR zanL;}*c# zy;~h>VLReZCNC$1{cMYDN2hqiJ!yg^b-`(bJ?~4{Rtuh#a@J|t*fdDUat@h>l`k%=#=*YG!6!aLl(?MKSR15Xj`8Cs=##ValHa@!RB?$d#I z(w-|V$n&n7wU(b9?t~e${|NF7EqSahh`r&1t@DWW7U~%FvW*stf{rZV>aG->L1mEq z-ewD*>CIM)nv|Gb1x@rOyEA=;7E!8j)~UQ#g17z%yA>u>Bf!@5P3Myr)&8d*;h2|E zNRXk`)95+kEw-f*w)fEklcf&@b5$8yVl0+Mcv~8YR0&Zo@}R9GlGC^s)p}2g;!bs| zoIFRY(=O*xrwFp0pi}t)Tk03pl`pCbFnh=SuWg{1gUL#O!J3$s)dPX`M+y;#%1P6-Edkd1ql^ngd19#`lGYUw#bEoh$0800B>F9% zskuC+xRkXD!#GoSmXlCLAd|6(?6om0L8ctp=@bCdyQ_o7j-R4p-4Ixjn$ZDZ%r455 zGcd)>2DQ;XV-~8TbnR(d;S|6>>uoU%wn6}=2(DR6MCGK?@$g_$2lT|0>cpf?EN{Q7 zf{ay~9Ck{96D62pUE3?nh-$YK7_JK!-6#b@%SXdduL5o5V17kgm})hnBDx4{UwF=i zKr%Rr5kUM`e~EajfR2PEJrP}+n0pm>T*z}R(;TqZQv`B`?~MoMHT--Dk$lpHHx*c+ zKr0RFz^QP?fkr`Bw?twTjt8wl#GL@KzP{_AlLKmJ3#4L>}aTSvbSsd6-;yCVoN}R`YSveqUh(4AJxp3safi^z1J2aDsORReKB#4R_b29#=-$1% Qy9f6?zTt$ASz_J$KLYLE4FCWD literal 0 HcmV?d00001 diff --git a/samples/laptop_profile_report.md b/samples/laptop_profile_report.md new file mode 100644 index 0000000..cf171ce --- /dev/null +++ b/samples/laptop_profile_report.md @@ -0,0 +1,152 @@ +# Kernel performance report + +**Host** notebook · kernel 6.8.0-138-generic · 8 CPUs (4 cores, 1 socket(s), SMT on) · 1 NUMA node(s) · L1D 48 KiB, L1I 32 KiB, L2 1280 KiB, L3 8192 KiB + +**Window** 1.201 s, 8 samples, recorder cost 1.379 ms/sample · **target** pid 2579584 (python3) · `python3 -c import time; x=[bytearray(1<<20) for _ in range(150)]; time.sleep(1)` + +## Findings + +No warnings: nothing crossed a threshold in this window. + +### 🔵 No hardware counters in this recording; cache and TLB efficiency can't be measured + +**Action:** Use `kptk profile -- ` with perf_event_paranoid <= 2 (or CAP_PERFMON), or pass `--perf-stat` with saved `perf stat -x;` output. + +### 🔵 CPU frequency governor 'powersave' is active + +Evidence: `governors=['powersave']` + +**Action:** On intel_pstate/amd-pstate 'powersave' is dynamic and often fine; for latency-critical services compare against 'performance' and check C-state exit latency. + +### 🔵 kernel.perf_event_paranoid=4: unprivileged hardware counters are disabled + +Evidence: `perf_event_paranoid=4` + +**Action:** For profiling sessions grant CAP_PERFMON to the profiler, or temporarily set perf_event_paranoid=2 (own processes, user+kernel). + +### 🔵 Single NUMA node: locality analysis doesn't apply + +Evidence: `nodes=1` + +**Action:** On multi-socket or sub-NUMA-clustered servers, record there to get locality findings. + +## USE summary + +| resource | utilisation | saturation | errors | +|---|---:|---|---| +| CPU | 0.0833 | psi_some: 0.00788, runq_waiting_tasks: 0.0478 | steal_frac: 0 | +| Memory capacity | 0.853 | psi_some: 0, direct_reclaim_per_s: 0, swap_pages_per_s: 0 | oom_kills: 0 | +| NUMA interconnect | - | - | - | +| Huge pages | 0 | compact_stall_per_s: 0 | - | + +## Over time + +``` +cpu util ▆▃▄▄▄█▁ max 0.155 +runq waiting ▇▁▁▁▂█▁ max 0.13 +faults/s █▁▁▁▁▁▁ max 200,929 +major flt/s ▁▁▁▁▁▁▁ max 0 +mem pressure ▁▁▁▁▁▁▁ max 0 +``` + +## Metrics + +
cpu + +| metric | value | +|---|---:| +| utilization | 0.0833 | +| user_frac | 0.0556 | +| system_frac | 0.0267 | +| iowait_frac | 0.00321 | +| steal_frac | 0 | +| softirq_frac | 0.00107 | +| irq_frac | 0 | +| context_switches_per_s | 11,538 | +| procs_running_mean | 1.25 | +| procs_running_per_cpu | 0.156 | +| per_cpu_spread | 0.093 | +| runq_waiting_tasks_avg | 0.0478 | +| runq_wait_per_timeslice_ms | 0.00806 | +| psi_cpu_some | 0.00788 | +| process_comm | python3 | +| process_threads | 1 | +| process_cpus_used | 0.0666 | +| process_system_frac | 0.75 | +| process_runq_wait_share | 0.00171 | +| process_runq_wait_per_slice_ms | 0.0184 | +| process_voluntary_switches_per_s | 1.67 | +| process_involuntary_switches_per_s | 5 | +| process_cpus_allowed | 0-7 | + +
+ +
memory + +| metric | value | +|---|---:| +| faults_per_s | 43,415 | +| major_faults_per_s | 0 | +| swap_pages_per_s | 0 | +| direct_reclaim_stalls_per_s | 0 | +| pgscan_direct_per_s | 0 | +| pgscan_kswapd_per_s | 0 | +| oom_kills | 0 | +| workingset_refault_per_s | 35 | +| mem_total_kib | 16110848 | +| mem_available_min_frac | 0.147 | +| psi_memory_some | 0 | +| psi_memory_full | 0 | +| process_minor_faults_per_s | 32,501 | +| process_major_faults_per_s | 0 | +| process_rss_kib | 0 | +| process_anon_kib | 158500 | +| process_swap_kib | 0 | + +
+ +
numa + +| metric | value | +|---|---:| +| nodes | 1 | +| applicable | False | + +
+ +
cache + +| metric | value | +|---|---:| +| available | False | + +
+ +
hugepages + +| metric | value | +|---|---:| +| thp_enabled | madvise | +| thp_defrag | madvise | +| khugepaged_max_ptes_none | 511 | +| thp_fault_alloc_per_s | 0 | +| thp_collapse_per_s | 0 | +| thp_split_per_s | 0 | +| compact_stall_per_s | 0 | +| anon_huge_kib_system | 0 | +| process_anon_kib | 158500 | +| process_thp_coverage | 0 | +| process_hugetlb_kib | 0 | + +
+ +
config + +| metric | value | +|---|---:| +| swappiness | 60 | +| zone_reclaim_mode | 0 | +| numa_balancing | 0 | +| perf_event_paranoid | 4 | + +
diff --git a/samples/numa_misplaced.jsonl.gz b/samples/numa_misplaced.jsonl.gz new file mode 100644 index 0000000000000000000000000000000000000000..11b7586a8f40a820e4a359cb6d8d572e0675a98b GIT binary patch literal 8094 zcmb_gdpy(q`>vi2l(3RRVo!$hJWPzJv3-%VC@QIta!NVnFo)SHidZzq9QQbsN>N5) zvJg3LG>05!IpjRUn9UA;)6?(yeV^Cw-~PGxn!R`3_xt)>_kG<1Ob`)q3aInhw&54z z>*nfvBPigO+jS3juuqV`->oh4&dxzY7ekTcva2&{FIqssuHUu=qya;dtyC&o`}#<# znsXY`r;W6)$GcltDKdHlYgZOclzi>=r$(+1UI;XpOZym9`j_9G(<^@kq6imJS-LW1 zkur$R+W~Pl)C$MsdHCr7^g*RP8MFY)<3VVb>TsTWwFaXW`sp2q|YYdvs;N@7N7Jn}yHg zZ$?qw9bCpn-vzId*?M1!D@M?_a67LTb}I{}!{qRboZr5_sJ@Gygr;)oV?M`m;$;o( z3Ejb87Fy7Ws%u2narp=weQC&&5IdLGRNU0N?nuZQrtMv4>>RIm*UuV$SMtr``>Raa zN_MPg`8;_iv);XCy%Wq1=QL7#$HuQ7mp`4=bZv)o=wB256iW(DuW{*Wa6d8IrOaYb zdV$wNAe4Bp-4Yf-LpYv5@a~3DjgM+HV%)QG7g`Pj7l&yG1_|<4;27!C{`*+B!LGA4 z)tM50YZrEBXK~gU=#kQc)$6fz$9_jdv?~|yfuFW((4+}!w|#Y*t_hdjn{CQnEw3gV zBrTLZOIh-E&iZg$pKVgru9;UC(WG7m;~{l3;S0_2aiRFoN7><&qB3oNX$fwL`=E;S z8gqb+3Ph&;ws?QT!?Atl*2%mPR87{0jSAVljRPm-#o!f32G_S-c&(K~DqK%zOm%)c znKQ`PmS2@12&+Ifwt`1@%*7D50X7hoA_U+%w~9_^a6jpv;Z1IzF{mvQlW(<(9_3E@ z=v4cmI!l{J&nPnTm}4aR&@53r{Wo96B%iKShAQ)m0+LGSP+>-ozAb1s&Tv(8o;KR? zSu2Beqb}?#Rgwi;KY{LSXgR4*tu=ateeMIlrTTuRm0Z>ch&hmmY`_kZG!i!q<*1mf zo4(x8j3K>DT1mFXnW6*n3!hD{E7x6U@u$>z?Rz?MM#-3O?%Lb_l*_-1S^NugW)n~L z%?~_GIzY8WWZWa>C!VYZ$R-2O+%5wB2an<&qo&>P4_6oAZ~AAsOXW>@ZE-iOHr{zSY|fq5S+7r3N7_D_j%)+vAtB~Yn@*~-x`yG_$D z?{(w?Ew#sNRY4>xv)jMPwMw+b&3XY-um*oAUHoiiw(Kp}y5C(nDFa*UU{)kD*kYlm z0Sd3dOe|?9BiiB=%`c8VPsSRYHd~gGYu&6|>Ghd&trmHzpzvhO#E}&2u$9^F-{jQ9 z+br^DpsFC#JEz_xreUPIwe6K#EC8Ay(_)MTpaP3kx?~m~ET=Z+Q8b+jFid>Ya8?Cm znuU>4)3y&5<}tmLFU+HK+ALmKPEA@c~dT0V@$g4MP(YfEXhxS;|%UyS(={=px5_95*SA(pL-tHM83 zV9p6)o7ydUf5OJ6WB+hAdl@G8;-AN5k){edACEcrf3Q|5*gq`IUfRmN0MA)(*Qi$s z6Gc4a?K8c@N*-kdG|BXw{N35LX8wy2mpQgny7yC2qD@i{_1Zq=calO8tAw<%Dy)@0RhJa`H&cp@8OeyLAk;9#RZ)F`(4kBR-(rTNFw-qNkC@ ziaP<}%I*{03du=Ee8X0Up|97EniaWI; zl_%!A6)OHI^j1<)g<;J&>wgP1TfnAQo1=uF?IVS6B(s#X8QY9_u#Xd`2v=2gmPC<7 zrxvk~>&+1&u8D=%u1`~oZjztOjrRemXL&y!kgfN=akfRyLFj}Sa&1U6Cfj3?t zrHZ8w;)7x8W1L0oL)oyxvwys?o(91Rl2S3Sea7pH$C>A@N>j{{w!oc6QOe4zqrMMb zepD|>V!wgyqpVxR&ovI&T)69W6}a>;O4*EsO&|C?W6V8fO>b0sm!uuqV0Aa=;xDcpdm3d%|B_gFw={rw@EpSqA`T%&ndQMe|YTjx!yvO3#@iOMyG< zg^1j<17dD>o!$YLMp4SoxL7HMxd-Io4$rOVPe*uI*(LK!&A{H7i6(Vg(x9=+?AIb2 zU!CL~p>pU?u{><@vbk{wu(I=t^phQY0B$ALs0S$Ogzh=s=yWxmWsV#MJ`6`I?+~0F zvhlv_Gyz=7L@PfKVAI#lqfUlvS@S5@;tcUENHIv^S&fQ4JP2%&oca;If7?dKt;xzR zbq1MT3*Wmeuxa5!@N$F3`1kPfae)nn3sGAjcX;03)A>b!vExFt=;VqDxSif$p63bl zC9$d_N)B2v-6ArBO*}au)61#@C#r>R@J!s(X)ip_q5S@x9Fq!ngLK^YJdAFRn) zd)c?(-egW2<1z4Yh`^?U4bd1UQy$e8Q#&G4^4BmO4Uf$VoxSzfU~Y?1R_}45qo^;Q zb$dF?`4}>b^7{}u<`(=0Wa7SOH>%m3n<-28vai5ZWUd(F-QnXYd>b1UL}S1%&(@_w zn;$H_Jf^h&Sg;w$)mvu93t5Cy!=hAAtM<1@FI;0@s8OrA{#6QV)3Lu~T6Oi^kj8)&aJ40}t7(eLpv63Y;k7IMt* z@SyDQV><0i5ZGtC=tJ=Rw|O1D4;s4Eo5^fh`2OoWoA@P2Ri>B;UJ5?$z_YPgf@n06 z9g5jE(#MKT<8h|$>-X<>$H@-fXUhS{ysYAIqK8q^$AhkUr;mCeAK-S%p;X=3PG7|m z@||LEO8}Ir89P&I)XNQ`^}tgU)qI4V`Q)3|rF*zy+eM0cy`BGIZJ73NCjPp54?+P# zp4gfDM!k$3csdqL$l*@%m(3hw zJWLW3qD3MX>BFrzA={Y{7@7RZ08Uxvsc(+97pM!;#tu!B*ZwG})~ZNYg~0mAsTbk< zukl}DggMmv$?Pley|?)`EvtC#%^X+ZojD~QayUGLSk{~XsAmPrE7GR6u8V3k?Mz5)Rlk|jUhQ7!<( z<``nam&0=mZDAgg`aEN-Q2^GOXCycLJoH6)h|btY5r8%5jPm_^9zoi9&`tAkfQVlR zx}M`aO>2r3I9pBARAK}Q!u#vvdx!~kd}wqJ(b|F!mF;{K==l~c1LPtezC~w9aS_Kh zbKl+o!`2x4f;PY!Ym5M&d`r6xyJ!8Kvt_+c?>K$*@*0?X4tqbp-2E67;F(#0@@B_=^d6OeUe8dTaQ~6&H0GnX(qwGbxzlU=u$DN5435^Sg6$y)b94itL_oi4R zvMmt-U`e^z{W~yDeI@wKo2Ps?sIB*yOaETh?(#M~2?-6aIz-j|P>krTN@3-`byCkV zO|p}h=5zSU>cK4kYTIykg=S8yqzJp=ikAWT<1`}arBe~LjFJ&bwq68b9;aa^JqX7 zB6`EwA36ArtuKF?MZ}FTq{Nzy=ncu^f#@$;uH(^~)YHJ7jOI?#$*P3Wt6zwUo!O(y z37tS`;4b`JaVKTQd85(07SEax_;sOy+iQt~qo$hhb$j@{Dd94A2fp4HKTUh&Hx=!H zXBMV2yyofzK5LD8nx-^|V;o2JJ^(h{DAh;Fp71xDVw5 ztw_yRXZH5bL9>B~sN)j2yoIULk!Ocpa%~%@O6nUm%JoDX^XVCa8S4z+I+u5F!QOIB z@rlphw<%LN#ry;K;sf~H&DdzKx3)FRG>Th@(-R#8-3*HGO>Fo+8AKfqMCe^oj~Mdz zd&cw(h7KU5eJ95~7rS27G^F>s4f)F~)ceB~Y%>_yoQUh^rR^RY5MAxeJAs3wS8KtG z?0rt>Imo77N=dfKQ4{Ch^Q!|11oLFj%P!*Oz{NX*wV}NC>jmHNjW?%OLFNNYzt*+Z zy|AgBBf-3Nd{1{9>$K&GY5qjPSg)(vI1J zI~(&3lM6U9>>!FY)m0Wq^gba!tXoC{578#*6XcSOnhFsbv@VxNrs7Z>6{NE}souYf zoiV___EZv>J(reyaCf=m>(RAi3N*X)XihOIQf1H;c}SyC_c}oqDDhs+}B{LRa>ZeWwQIv^2SbtQP(>2_gicDF5K(y;K=hM!@~xp%-P-&C&m&%J@SLn zl;&J1QJDM&AhE(wY_M>VLR}Gbv2qJjL&H%B&%ezV8FLH<%El(q+Gn~qZ??kO;c@E&ph0C!Is z68@LP5;CH=!WSyBm1f(}pkE#!?XIVpKyMrJ$E7z@nxk*-;WBc}nwY&=J$G4l#!R4j=ZrF>2nzry0^p=nU6J+TIOe9wznXn!)9$ zf?J3~=6oti{3SG}2IPp4niVfwINGa_R#Vsqku|Ex3z@zVn}?Ww7@HS9{USCma=JA( zFKTwc>(hxcFI)uc#lJ&tUR1IBh7&qNvys+G;5mOetuJod%O6~Xcj{6^OD3Y zqg3qUDT!`wZ*gOk#M?>$S>TsU6rMqZg_yrsc;2H> znO86JW%ZSb9dxDE#Li%)!^F;TW!S_HwleQPhJi@o|8FGs;2A6PP!%qDoxo{dM3jKh z8CU^0kASW!XCj6MEM36bS`}tJ=j?MuZeG$Pl zME5%m{GhH~NL{C?01V|*eaN)e2?x`d-&y5r^NhMJ=>8Xcs`j7NEQGadrNHm(rqbw+ zp;qiTAgN(CjyQDwS30D`I-#74;P)>g+CgiKdxZW4l?yXvp`WQ*=qf_=ta$N4ORvKH zn#K~SRfzf1h3BhcW{*p~6e0%mrxYT_^1T!zX7i^MBbM_wg(eqBnj0wI)Zf2Cu@_Bk z0NqDh)oh^c5UNgQjsX*HAKp`pljttmYm#7LQeR-ir+)dS*c}lCpU#WDSjvz$uwKLNDWk7kX_6n@~MS zd))^XO(}-96ngD)P!J0NA?<^% za-m6(m5?zPI~_~-SdP!)T%cJ=anXAav-f1eQMKF21y}6PS;W^CJx`T7pImTh`!liI zfJw=_;(%GndU3#lp5vZRs3k9BNd3H)Epx;X(i-{y)$_CMkqp&f%az;(??-9wQA zFv4ppHM5a55a0C6g-4<_wLd9qCUJaBUlF+-e1c9+*`x%?F=&e8vUx13KkZSy7xMmFTrzk55@p3YOBES1q;y_ z8?#B>t09gL>AU%430BCQDU@^+uI>8z*EZ}5!efnrdc^V3XYrgDNF+7qg?Ho&>PhQ_ zT!nK8{Sfogg`+(RF4VIxp?`#!zxFzIFW7Ldiz05w=-Mw~$n0{EFkI_Ok=Qf|a$Jzh z<|Z$+jsFf%L#}PB054ehsXGs$?uPk<2h84=(ahduCbCj5dz`&%^@r?gxnpu=2Lez6 zznA}*?7|3Ax26|a$(!qN{FCU4c0BI-dVU2VRNd0j&XIp;dnz<+ME!lrGefVlga)C; zny(p>TB^n|%fGx^52N)ms6?*rWXaN3CW5*!pR<~BpdaV;^18qe@r%dYrrxTB?Rvh5 zm<$2krQ@bbJ`4yw_8HMS!K>B}#W$1)oXXGiLhF%e{J!BFK2;oz@UJ`A z{*$?Op-*fwH@l2b5N<6mk78qqRsU-em4h^2;B}$PI`;`jLCMsj;tHZq@e1|qb7*Z%;W@;)*~ClatJ`NH z`tpkvB1ZC$C`L@?7b`|A zNEdJsoS(vDei%5>8=-nC((;J{6G8Z;+)eSE>o6Rj@P4GOJ67PgBy@^uDzdWIgil;S z@Wbl{vpWNM(d&erNFXAalD}0482WXB2RJT55Py zq<`Ju`Y6Hl(T}#}V|mn(rho`q9Yg~Rj+E&%f!v518S$5Cv};Vumajv~Vi-eT6vohP z#;eH1K=e(5udn-x|75gVnAUR!t+B7${cR2{Rw2ihl`$Ni&Lky6FDHRd9Eyov?yar9 zG1=+P9?0w4CH?xo^*aU)w=uTxv5a1m9;pYk8KlgLJlJ)3{^EQWX=w|}xx{&c14pBD z^Bz=Xet7!Q1nL;V7#_d9YfTgV;9H~hyPIGb#uGnTz4lkd%^L! literal 0 HcmV?d00001 diff --git a/samples/numa_misplaced_report.json b/samples/numa_misplaced_report.json new file mode 100644 index 0000000..d22b7b3 --- /dev/null +++ b/samples/numa_misplaced_report.json @@ -0,0 +1,627 @@ +{ + "generated_at": 1790865497.7161434, + "meta": { + "tool": "kptk synth", + "host": "synthetic-2s", + "kernel": "6.8.0-synthetic", + "arch": "x86_64", + "interval_s": 1.0, + "pid": 4242, + "command": null, + "user_hz": 100, + "page_size": 4096, + "overruns": 0 + }, + "topology": { + "cpus": 32, + "packages": 2, + "physical_cores": 16, + "smt": true, + "numa_nodes": 2, + "node_cpus": { + "0": "0-7,16-23", + "1": "8-15,24-31" + }, + "distances": { + "0": [ + 10, + 21 + ], + "1": [ + 21, + 10 + ] + }, + "caches": [ + { + "level": 1, + "type": "Data", + "size_kib": 48, + "shared_by_cpus": 2, + "instances": 16 + }, + { + "level": 1, + "type": "Instruction", + "size_kib": 32, + "shared_by_cpus": 2, + "instances": 16 + }, + { + "level": 2, + "type": "Unified", + "size_kib": 2048, + "shared_by_cpus": 2, + "instances": 16 + }, + { + "level": 3, + "type": "Unified", + "size_kib": 32768, + "shared_by_cpus": 16, + "instances": 2 + } + ], + "governors": [ + "performance" + ] + }, + "metrics": { + "cpu": { + "utilization": 0.4, + "user_frac": 0.32, + "system_frac": 0.08, + "iowait_frac": 0.0, + "steal_frac": 0.0, + "softirq_frac": 0.0, + "irq_frac": 0.0, + "per_cpu_utilization": { + "0": 0.4, + "1": 0.4, + "2": 0.4, + "3": 0.4, + "4": 0.4, + "5": 0.4, + "6": 0.4, + "7": 0.4, + "8": 0.4, + "9": 0.4, + "10": 0.4, + "11": 0.4, + "12": 0.4, + "13": 0.4, + "14": 0.4, + "15": 0.4, + "16": 0.4, + "17": 0.4, + "18": 0.4, + "19": 0.4, + "20": 0.4, + "21": 0.4, + "22": 0.4, + "23": 0.4, + "24": 0.4, + "25": 0.4, + "26": 0.4, + "27": 0.4, + "28": 0.4, + "29": 0.4, + "30": 0.4, + "31": 0.4 + }, + "context_switches_per_s": 50000.0, + "procs_running_mean": 13.0, + "procs_running_per_cpu": 0.40625, + "per_cpu_spread": 0.0, + "runq_waiting_tasks_avg": 0.05, + "runq_wait_per_timeslice_ms": 0.0015625, + "runq_waiting_tasks_per_cpu": { + "0": 0.002, + "1": 0.002, + "2": 0.002, + "3": 0.002, + "4": 0.002, + "5": 0.002, + "6": 0.002, + "7": 0.002, + "8": 0.002, + "9": 0.002, + "10": 0.002, + "11": 0.002, + "12": 0.002, + "13": 0.002, + "14": 0.002, + "15": 0.002, + "16": 0.002, + "17": 0.002, + "18": 0.002, + "19": 0.002, + "20": 0.002, + "21": 0.002, + "22": 0.002, + "23": 0.002, + "24": 0.002, + "25": 0.002, + "26": 0.002, + "27": 0.002, + "28": 0.002, + "29": 0.002, + "30": 0.002, + "31": 0.002 + }, + "psi_cpu_some": 0.01, + "process_comm": "pgdb", + "process_threads": 32, + "process_cpus_used": 4.0, + "process_system_frac": 0.15, + "process_cpus_seen": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ], + "process_node_residency": { + "0": 1.0 + }, + "process_runq_wait_share": 0.0199999999853, + "process_runq_wait_per_slice_ms": 0.0163265306, + "process_voluntary_switches_per_s": 2000.0, + "process_involuntary_switches_per_s": 100.0, + "process_cpus_allowed": "0-31" + }, + "memory": { + "faults_per_s": 20000.0, + "major_faults_per_s": 0.0, + "swap_pages_per_s": 0.0, + "direct_reclaim_stalls_per_s": 0.0, + "pgscan_direct_per_s": 0.0, + "pgscan_kswapd_per_s": 0.0, + "oom_kills": 0, + "workingset_refault_per_s": 0.0, + "mem_total_kib": 134217728, + "mem_available_min_frac": 0.25849999487400055, + "psi_memory_some": 0.0, + "psi_memory_full": 0.0, + "process_minor_faults_per_s": 2000.0, + "process_major_faults_per_s": 0.0, + "process_rss_kib": 50331648, + "process_anon_kib": 50331648, + "process_swap_kib": 0 + }, + "numa": { + "nodes": 2, + "distances": { + "0": [ + 10, + 21 + ], + "1": [ + 21, + 10 + ] + }, + "applicable": true, + "remote_alloc_frac": 0.35, + "policy_miss_frac": 0.0, + "per_node_alloc": { + "0": { + "local": 162500, + "other": 87500, + "miss": 0 + }, + "1": { + "local": 162500, + "other": 87500, + "miss": 0 + } + }, + "numa_balancing": "1", + "hint_fault_local_frac": 0.25, + "pages_migrated_per_s": 0.0, + "node_free_frac": { + "0": 0.45, + "1": 0.02 + }, + "process_memory_by_node": { + "0": 0.15, + "1": 0.85 + }, + "process_memory_policies": { + "default": 2 + }, + "process_expected_distance": 19.35, + "process_remote_access_frac_est": 0.85, + "process_placement_mismatch": 0.85 + }, + "cache": { + "caches": [ + { + "level": 1, + "type": "Data", + "size_kib": 48, + "shared_by_cpus": 2 + }, + { + "level": 1, + "type": "Instruction", + "size_kib": 32, + "shared_by_cpus": 2 + }, + { + "level": 2, + "type": "Unified", + "size_kib": 2048, + "shared_by_cpus": 2 + }, + { + "level": 3, + "type": "Unified", + "size_kib": 32768, + "shared_by_cpus": 16 + } + ], + "available": true, + "ipc": 0.6, + "llc_miss_ratio": 0.4777777777777778, + "llc_mpki": 17.916666666666668, + "branch_miss_ratio": 0.005, + "branch_mpki": 0.8333333333333334, + "l1d_miss_ratio": null, + "dtlb_miss_ratio": 0.03, + "dtlb_mpki": 10.0, + "backend_stall_frac": null, + "frontend_stall_frac": null, + "min_running_frac": 1.0 + }, + "hugepages": { + "thp_enabled": "always", + "thp_defrag": "always", + "khugepaged_max_ptes_none": "511", + "thp_fault_alloc_per_s": 50.0, + "thp_fallback_frac": 0.4444444444444444, + "thp_collapse_per_s": 0.0, + "thp_split_per_s": 0.0, + "compact_stall_per_s": 12.0, + "compact_fail_frac": 0.5, + "hugetlb_pools": { + "2048": { + "nr_hugepages": 2048, + "free_hugepages": 2048, + "resv_hugepages": 0, + "surplus_hugepages": 0 + } + }, + "anon_huge_kib_system": 0, + "process_anon_kib": 50331648, + "process_thp_coverage": 0.03999998172124227, + "process_hugetlb_kib": 0 + }, + "config": { + "swappiness": "60", + "zone_reclaim_mode": "0", + "numa_balancing": "1", + "perf_event_paranoid": "2", + "governors": [ + "performance" + ] + } + }, + "use": [ + { + "resource": "CPU", + "utilization": 0.4, + "saturation": { + "psi_some": 0.01, + "runq_waiting_tasks": 0.05 + }, + "errors": { + "steal_frac": 0.0 + } + }, + { + "resource": "Memory capacity", + "utilization": 0.7415000051259995, + "saturation": { + "psi_some": 0.0, + "direct_reclaim_per_s": 0.0, + "swap_pages_per_s": 0.0 + }, + "errors": { + "oom_kills": 0 + } + }, + { + "resource": "NUMA interconnect", + "utilization": 0.35, + "saturation": { + "pages_migrated_per_s": 0.0 + }, + "errors": { + "policy_miss_frac": 0.0 + } + }, + { + "resource": "Huge pages", + "utilization": 0.03999998172124227, + "saturation": { + "compact_stall_per_s": 12.0 + }, + "errors": { + "thp_fallback_frac": 0.4444444444444444 + } + } + ], + "findings": [ + { + "id": "numa.process_misplaced", + "domain": "numa", + "severity": "critical", + "title": "About 85% of the process's memory accesses are likely remote: threads ran mostly on node 0, memory sits mostly on node 1", + "evidence": { + "cpu_node_residency": { + "0": 1.0 + }, + "memory_by_node": { + "0": 0.15, + "1": 0.85 + }, + "placement_mismatch": 0.85, + "expected_slit_distance": 19.35 + }, + "recommendation": "Start it with numactl --cpunodebind=1 --membind=1, or migratepages 4242 1 0.", + "why": "Typical cause: memory was first-touched by an initialisation thread on one node, then worker threads were scheduled on another." + }, + { + "id": "cache.memory_bound", + "domain": "cache", + "severity": "warning", + "title": "Likely memory-bound: IPC 0.60, LLC 17.9 misses per 1k instructions; anonymous working set 49152 MiB vs 32 MiB L3", + "evidence": { + "ipc": 0.6, + "llc_mpki": 17.9167, + "llc_miss_ratio": 0.4778 + }, + "recommendation": "Improve locality: tile or block loops to fit the LLC, use struct-of-arrays for scanned fields, prefer contiguous containers over pointer-chasing, and prefetch for irregular access.", + "why": "Low IPC together with many LLC misses means the core is waiting on DRAM. Top-down analysis (perf stat --topdown) can confirm it on supported CPUs." + }, + { + "id": "cache.tlb_pressure", + "domain": "cache", + "severity": "warning", + "title": "dTLB miss rate 3.0% (10.00 per 1k instructions): page walks are significant", + "evidence": { + "dtlb_miss_ratio": 0.03 + }, + "recommendation": "Back the hot heap with huge pages (madvise(MADV_HUGEPAGE), THP, or hugetlbfs). A 2 MiB page covers 512x the reach of a 4 KiB TLB entry. See the hugepages findings.", + "why": "With a few thousand TLB entries x 4 KiB, reach is only a few MiB; larger random-access working sets miss constantly." + }, + { + "id": "hp.sync_compaction", + "domain": "hugepages", + "severity": "warning", + "title": "THP enabled=always with defrag=always: page faults may stall on synchronous compaction", + "evidence": { + "enabled": "always", + "defrag": "always" + }, + "recommendation": "Use defrag=defer+madvise: background compaction for everyone, synchronous only for regions that asked with MADV_HUGEPAGE.", + "why": "Under fragmentation every anonymous fault can enter direct compaction, a classic source of multi-ms latency spikes in databases and JVMs." + }, + { + "id": "hp.thp_fallback", + "domain": "hugepages", + "severity": "warning", + "title": "44% of huge-page faults fell back to 4 KiB pages", + "evidence": { + "thp_fault_alloc": 500, + "thp_fault_fallback": 400 + }, + "recommendation": "Physical memory is fragmented. Reserve hugetlbfs pages at boot for critical workloads, raise vm.min_free_kbytes, or trigger compaction off-peak (echo 1 > /proc/sys/vm/compact_memory).", + "why": "" + }, + { + "id": "hp.compaction_stalls", + "domain": "hugepages", + "severity": "warning", + "title": "12.0 direct compaction stalls/s", + "evidence": { + "compact_stall_per_s": 12.0, + "compact_fail_frac": 0.5 + }, + "recommendation": "Tasks are compacting memory synchronously to get contiguous pages. Prefer THP defrag=defer+madvise and keep free-memory headroom.", + "why": "" + }, + { + "id": "hp.low_coverage", + "domain": "hugepages", + "severity": "warning", + "title": "Only 4% of the process's 49152 MiB anonymous memory is backed by huge pages and dTLB miss rate is 3.0%", + "evidence": { + "anon_kib": 50331648, + "anon_huge_kib": 2013265, + "thp_enabled": "always" + }, + "recommendation": "Try enabling THP (or hugetlbfs for predictable latency). For jemalloc/tcmalloc there are allocator options to request huge pages. Measure: the benefit depends on random-access TLB pressure.", + "why": "" + }, + { + "id": "numa.remote_alloc", + "domain": "numa", + "severity": "warning", + "title": "35% of page allocations landed on a node other than the allocating CPU's", + "evidence": { + "remote_alloc_frac": 0.35, + "policy_miss_frac": 0 + }, + "recommendation": "Bind CPU and memory together (numactl --cpunodebind=N --membind=N, or cpuset.mems) and check for interleave or preferred policies set by the runtime.", + "why": "Remote DRAM costs roughly 1.3-2x local latency and shares inter-socket link bandwidth." + }, + { + "id": "numa.access_locality", + "domain": "numa", + "severity": "warning", + "title": "Only 25% of sampled memory accesses were node-local (AutoNUMA hinting faults)", + "evidence": { + "hint_faults": 400000, + "local": 100000 + }, + "recommendation": "Threads and their data are on different nodes. Partition data per node, or pin threads to the node holding their memory.", + "why": "" + }, + { + "id": "numa.node_exhausted", + "domain": "numa", + "severity": "warning", + "title": "Node 1 is nearly full (2.0% free): new allocations will spill to node(s) [0] or trigger reclaim", + "evidence": { + "node": 1, + "free_frac": 0.02, + "zone_reclaim_mode": "0" + }, + "recommendation": "Rebalance placement or interleave large shared buffers. With zone_reclaim_mode=0 (default) the kernel goes remote rather than reclaim locally.", + "why": "" + }, + { + "id": "hp.hugetlb_idle", + "domain": "hugepages", + "severity": "info", + "title": "4 GiB of 2 MiB hugetlb pages are reserved in the pool but unused", + "evidence": { + "page_size_kib": 2048, + "nr": 2048, + "free": 2048, + "reserved": 0 + }, + "recommendation": "hugetlb pages are unavailable to the rest of the system. Shrink vm.nr_hugepages, or confirm the consumer (DPDK, a DB) starts later.", + "why": "" + } + ], + "counters": { + "cycles": { + "value": 40000000000.0, + "raw": 40000000000, + "running_frac": 1.0 + }, + "instructions": { + "value": 24000000000.0, + "raw": 24000000000, + "running_frac": 1.0 + }, + "cache-references": { + "value": 900000000.0, + "raw": 900000000, + "running_frac": 1.0 + }, + "cache-misses": { + "value": 430000000.0, + "raw": 430000000, + "running_frac": 1.0 + }, + "branches": { + "value": 4000000000.0, + "raw": 4000000000, + "running_frac": 1.0 + }, + "branch-misses": { + "value": 20000000.0, + "raw": 20000000, + "running_frac": 1.0 + }, + "dTLB-loads": { + "value": 8000000000.0, + "raw": 8000000000, + "running_frac": 1.0 + }, + "dTLB-load-misses": { + "value": 240000000.0, + "raw": 240000000, + "running_frac": 1.0 + } + }, + "timeseries": { + "t": [ + 1.0, + 2.0, + 3.0, + 4.0, + 5.0, + 6.0, + 7.0, + 8.0, + 9.0, + 10.0 + ], + "cpu_util": [ + 0.4, + 0.4, + 0.4, + 0.4, + 0.4, + 0.4, + 0.4, + 0.4, + 0.4, + 0.4 + ], + "runq_waiting": [ + 0.05, + 0.05, + 0.05, + 0.05, + 0.05, + 0.05, + 0.05, + 0.05, + 0.05, + 0.05 + ], + "faults_per_s": [ + 20000.0, + 20000.0, + 20000.0, + 20000.0, + 20000.0, + 20000.0, + 20000.0, + 20000.0, + 20000.0, + 20000.0 + ], + "major_faults_per_s": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "psi_mem_some": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "data_quality": { + "samples": 11, + "duration_s": 10.0, + "mean_sample_cost_ms": 0.4, + "overruns": 0, + "missing": [], + "process_samples": 11 + } +} \ No newline at end of file diff --git a/samples/numa_misplaced_report.md b/samples/numa_misplaced_report.md new file mode 100644 index 0000000..55514b0 --- /dev/null +++ b/samples/numa_misplaced_report.md @@ -0,0 +1,222 @@ +# Kernel performance report + +**Host** synthetic-2s · kernel 6.8.0-synthetic · 32 CPUs (16 cores, 2 socket(s), SMT on) · 2 NUMA node(s) · L1D 48 KiB, L1I 32 KiB, L2 2048 KiB, L3 32768 KiB + +**Window** 10.0 s, 11 samples, recorder cost 0.4 ms/sample · **target** pid 4242 (pgdb) + +## Findings + +### 🔴 About 85% of the process's memory accesses are likely remote: threads ran mostly on node 0, memory sits mostly on node 1 + +Evidence: `cpu_node_residency={0: 1}`, `memory_by_node={0: 0.15, 1: 0.85}`, `placement_mismatch=0.85`, `expected_slit_distance=19.4` + +_Why it matters:_ Typical cause: memory was first-touched by an initialisation thread on one node, then worker threads were scheduled on another. + +**Action:** Start it with numactl --cpunodebind=1 --membind=1, or migratepages 4242 1 0. + +### 🟠 Likely memory-bound: IPC 0.60, LLC 17.9 misses per 1k instructions; anonymous working set 49152 MiB vs 32 MiB L3 + +Evidence: `ipc=0.6`, `llc_mpki=17.9`, `llc_miss_ratio=0.478` + +_Why it matters:_ Low IPC together with many LLC misses means the core is waiting on DRAM. Top-down analysis (perf stat --topdown) can confirm it on supported CPUs. + +**Action:** Improve locality: tile or block loops to fit the LLC, use struct-of-arrays for scanned fields, prefer contiguous containers over pointer-chasing, and prefetch for irregular access. + +### 🟠 dTLB miss rate 3.0% (10.00 per 1k instructions): page walks are significant + +Evidence: `dtlb_miss_ratio=0.03` + +_Why it matters:_ With a few thousand TLB entries x 4 KiB, reach is only a few MiB; larger random-access working sets miss constantly. + +**Action:** Back the hot heap with huge pages (madvise(MADV_HUGEPAGE), THP, or hugetlbfs). A 2 MiB page covers 512x the reach of a 4 KiB TLB entry. See the hugepages findings. + +### 🟠 THP enabled=always with defrag=always: page faults may stall on synchronous compaction + +Evidence: `enabled=always`, `defrag=always` + +_Why it matters:_ Under fragmentation every anonymous fault can enter direct compaction, a classic source of multi-ms latency spikes in databases and JVMs. + +**Action:** Use defrag=defer+madvise: background compaction for everyone, synchronous only for regions that asked with MADV_HUGEPAGE. + +### 🟠 44% of huge-page faults fell back to 4 KiB pages + +Evidence: `thp_fault_alloc=500`, `thp_fault_fallback=400` + +**Action:** Physical memory is fragmented. Reserve hugetlbfs pages at boot for critical workloads, raise vm.min_free_kbytes, or trigger compaction off-peak (echo 1 > /proc/sys/vm/compact_memory). + +### 🟠 12.0 direct compaction stalls/s + +Evidence: `compact_stall_per_s=12`, `compact_fail_frac=0.5` + +**Action:** Tasks are compacting memory synchronously to get contiguous pages. Prefer THP defrag=defer+madvise and keep free-memory headroom. + +### 🟠 Only 4% of the process's 49152 MiB anonymous memory is backed by huge pages and dTLB miss rate is 3.0% + +Evidence: `anon_kib=50331648`, `anon_huge_kib=2013265`, `thp_enabled=always` + +**Action:** Try enabling THP (or hugetlbfs for predictable latency). For jemalloc/tcmalloc there are allocator options to request huge pages. Measure: the benefit depends on random-access TLB pressure. + +### 🟠 35% of page allocations landed on a node other than the allocating CPU's + +Evidence: `remote_alloc_frac=0.35`, `policy_miss_frac=0` + +_Why it matters:_ Remote DRAM costs roughly 1.3-2x local latency and shares inter-socket link bandwidth. + +**Action:** Bind CPU and memory together (numactl --cpunodebind=N --membind=N, or cpuset.mems) and check for interleave or preferred policies set by the runtime. + +### 🟠 Only 25% of sampled memory accesses were node-local (AutoNUMA hinting faults) + +Evidence: `hint_faults=400000`, `local=100000` + +**Action:** Threads and their data are on different nodes. Partition data per node, or pin threads to the node holding their memory. + +### 🟠 Node 1 is nearly full (2.0% free): new allocations will spill to node(s) [0] or trigger reclaim + +Evidence: `node=1`, `free_frac=0.02`, `zone_reclaim_mode=0` + +**Action:** Rebalance placement or interleave large shared buffers. With zone_reclaim_mode=0 (default) the kernel goes remote rather than reclaim locally. + +### 🔵 4 GiB of 2 MiB hugetlb pages are reserved in the pool but unused + +Evidence: `page_size_kib=2048`, `nr=2048`, `free=2048`, `reserved=0` + +**Action:** hugetlb pages are unavailable to the rest of the system. Shrink vm.nr_hugepages, or confirm the consumer (DPDK, a DB) starts later. + +## USE summary + +| resource | utilisation | saturation | errors | +|---|---:|---|---| +| CPU | 0.4 | psi_some: 0.01, runq_waiting_tasks: 0.05 | steal_frac: 0 | +| Memory capacity | 0.742 | psi_some: 0, direct_reclaim_per_s: 0, swap_pages_per_s: 0 | oom_kills: 0 | +| NUMA interconnect | 0.35 | pages_migrated_per_s: 0 | policy_miss_frac: 0 | +| Huge pages | 0.04 | compact_stall_per_s: 12 | thp_fallback_frac: 0.444 | + +## Over time + +``` +cpu util ▁▁▁▁▁▁▁▁▁▁ max 0.4 +runq waiting ▁▁▁▁▁▁▁▁▁▁ max 0.05 +faults/s ▁▁▁▁▁▁▁▁▁▁ max 20,000 +major flt/s ▁▁▁▁▁▁▁▁▁▁ max 0 +mem pressure ▁▁▁▁▁▁▁▁▁▁ max 0 +``` + +## Metrics + +
cpu + +| metric | value | +|---|---:| +| utilization | 0.4 | +| user_frac | 0.32 | +| system_frac | 0.08 | +| iowait_frac | 0 | +| steal_frac | 0 | +| softirq_frac | 0 | +| irq_frac | 0 | +| context_switches_per_s | 50,000 | +| procs_running_mean | 13 | +| procs_running_per_cpu | 0.406 | +| per_cpu_spread | 0 | +| runq_waiting_tasks_avg | 0.05 | +| runq_wait_per_timeslice_ms | 0.00156 | +| psi_cpu_some | 0.01 | +| process_comm | pgdb | +| process_threads | 32 | +| process_cpus_used | 4 | +| process_system_frac | 0.15 | +| process_runq_wait_share | 0.02 | +| process_runq_wait_per_slice_ms | 0.0163 | +| process_voluntary_switches_per_s | 2,000 | +| process_involuntary_switches_per_s | 100 | +| process_cpus_allowed | 0-31 | + +
+ +
memory + +| metric | value | +|---|---:| +| faults_per_s | 20,000 | +| major_faults_per_s | 0 | +| swap_pages_per_s | 0 | +| direct_reclaim_stalls_per_s | 0 | +| pgscan_direct_per_s | 0 | +| pgscan_kswapd_per_s | 0 | +| oom_kills | 0 | +| workingset_refault_per_s | 0 | +| mem_total_kib | 134217728 | +| mem_available_min_frac | 0.258 | +| psi_memory_some | 0 | +| psi_memory_full | 0 | +| process_minor_faults_per_s | 2,000 | +| process_major_faults_per_s | 0 | +| process_rss_kib | 50331648 | +| process_anon_kib | 50331648 | +| process_swap_kib | 0 | + +
+ +
numa + +| metric | value | +|---|---:| +| nodes | 2 | +| applicable | True | +| remote_alloc_frac | 0.35 | +| policy_miss_frac | 0 | +| numa_balancing | 1 | +| hint_fault_local_frac | 0.25 | +| pages_migrated_per_s | 0 | +| process_expected_distance | 19.4 | +| process_remote_access_frac_est | 0.85 | +| process_placement_mismatch | 0.85 | + +
+ +
cache + +| metric | value | +|---|---:| +| available | True | +| ipc | 0.6 | +| llc_miss_ratio | 0.478 | +| llc_mpki | 17.9 | +| branch_miss_ratio | 0.005 | +| branch_mpki | 0.833 | +| dtlb_miss_ratio | 0.03 | +| dtlb_mpki | 10 | +| min_running_frac | 1 | + +
+ +
hugepages + +| metric | value | +|---|---:| +| thp_enabled | always | +| thp_defrag | always | +| khugepaged_max_ptes_none | 511 | +| thp_fault_alloc_per_s | 50 | +| thp_fallback_frac | 0.444 | +| thp_collapse_per_s | 0 | +| thp_split_per_s | 0 | +| compact_stall_per_s | 12 | +| compact_fail_frac | 0.5 | +| anon_huge_kib_system | 0 | +| process_anon_kib | 50331648 | +| process_thp_coverage | 0.04 | +| process_hugetlb_kib | 0 | + +
+ +
config + +| metric | value | +|---|---:| +| swappiness | 60 | +| zone_reclaim_mode | 0 | +| numa_balancing | 1 | +| perf_event_paranoid | 2 | + +
diff --git a/samples/parse_report.json b/samples/parse_report.json new file mode 100644 index 0000000..5a606a6 --- /dev/null +++ b/samples/parse_report.json @@ -0,0 +1,85 @@ +{ + "perf": { + "cycles": 1000000.0, + "instructions": 2500000.0, + "cache-references": 20000.0, + "cache-misses": 1000.0, + "context-switches": 120.0, + "cpu-migrations": 4.0, + "minor-faults": 80.0, + "major-faults": 2.0, + "task-clock": 150.0 + }, + "cache": { + "caches": [], + "available": true, + "ipc": 2.5, + "llc_miss_ratio": 0.05, + "llc_mpki": 0.4, + "branch_miss_ratio": null, + "branch_mpki": null, + "l1d_miss_ratio": null, + "dtlb_miss_ratio": null, + "dtlb_mpki": null, + "backend_stall_frac": null, + "frontend_stall_frac": null, + "min_running_frac": 1.0 + }, + "findings": [ + { + "id": "cache.efficient", + "domain": "cache", + "severity": "info", + "title": "IPC 2.50: the core pipeline is well utilised", + "evidence": { + "ipc": 2.5 + }, + "recommendation": "Gains now come from doing less work (algorithms, vectorisation), not from memory layout.", + "why": "" + } + ], + "software": { + "task-clock": 150.0, + "context-switches": 120.0, + "cpu-migrations": 4.0, + "minor-faults": 80.0, + "major-faults": 2.0 + }, + "vmstat": { + "r": 1.0, + "b": 0.0, + "swpd": 0.0, + "free": 100000.0, + "buff": 20000.0, + "cache": 300000.0, + "si": 0.0, + "so": 0.0, + "bi": 1.0, + "bo": 2.0, + "in": 100.0, + "cs": 120.0, + "us": 10.0, + "sy": 2.0, + "id": 88.0, + "wa": 0.0, + "st": 0.0 + }, + "numastat": { + "numa_hit": { + "node0": 10000.0, + "node1": 9000.0, + "total": 19000.0 + }, + "numa_miss": { + "node0": 100.0, + "node1": 300.0, + "total": 400.0 + }, + "numa_foreign": { + "node0": 0.0, + "node1": 0.0, + "total": 0.0 + } + }, + "policy_miss_frac": 0.020618556701030927 +} diff --git a/samples/report.json b/samples/report.json deleted file mode 100644 index 0762b23..0000000 --- a/samples/report.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "generated_at": "2026-07-22T16:39:36.545595+00:00", - "metrics": { - "cache_miss_rate_percent": 5.0, - "cache_misses": 1000.0, - "cache_references": 20000.0, - "context_switches": 120.0, - "context_switches_per_ms": 0.8, - "cpu_migrations": 4.0, - "instructions_per_cache_miss": 2500.0, - "major_fault_percent": 2.4390243902439024, - "major_faults": 2.0, - "minor_faults": 80.0, - "numa_local_allocations": 19000.0, - "numa_remote_allocations": 400.0, - "numa_remote_percent": 2.0618556701030926, - "total_faults": 82.0 - }, - "numastat": { - "numa_foreign": { - "node0": 0.0, - "node1": 0.0, - "total": 0.0 - }, - "numa_hit": { - "node0": 10000.0, - "node1": 9000.0, - "total": 19000.0 - }, - "numa_miss": { - "node0": 100.0, - "node1": 300.0, - "total": 400.0 - } - }, - "perf": { - "cache-misses": 1000.0, - "cache-references": 20000.0, - "context-switches": 120.0, - "cpu-migrations": 4.0, - "cycles": 1000000.0, - "instructions": 2500000.0, - "major-faults": 2.0, - "minor-faults": 80.0, - "task-clock": 150.0 - }, - "vmstat": { - "b": 0.0, - "bi": 1.0, - "bo": 2.0, - "buff": 20000.0, - "cache": 300000.0, - "cs": 120.0, - "free": 100000.0, - "id": 88.0, - "in": 100.0, - "r": 1.0, - "si": 0.0, - "so": 0.0, - "st": 0.0, - "swpd": 0.0, - "sy": 2.0, - "us": 10.0, - "wa": 0.0 - } -} diff --git a/scripts/benchmark_all.sh b/scripts/benchmark_all.sh deleted file mode 100755 index 20a5961..0000000 --- a/scripts/benchmark_all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -make -C "$root" sample-report -echo "Wrote $root/samples/report.json" diff --git a/scripts/collect_metrics.sh b/scripts/collect_metrics.sh deleted file mode 100755 index 34da7b4..0000000 --- a/scripts/collect_metrics.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ "$#" -eq 0 ]; then - echo "Usage: $0 COMMAND [ARG ...]" >&2 - exit 2 -fi - -exec python3 "$(dirname "$0")/../src/perf_runner.py" --command "$@" diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh deleted file mode 100755 index 092ae9e..0000000 --- a/scripts/install_dependencies.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -command -v perf >/dev/null || echo "Install linux-tools for live perf collection." -command -v numastat >/dev/null || echo "Install numactl for NUMA collection." -echo "No Python runtime dependencies are required; install pytest to run tests." diff --git a/scripts/run_microbenchmarks.sh b/scripts/run_microbenchmarks.sh new file mode 100755 index 0000000..8aa9d51 --- /dev/null +++ b/scripts/run_microbenchmarks.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Build and run the C microbenchmarks. CPU/memory heavy: run on an otherwise idle +# machine, never in CI. Results go to build/results/. +set -euo pipefail +cd "$(dirname "$0")/.." +make bench-build +mkdir -p build/results +./build/mem_latency | tee build/results/mem_latency_4k.csv +./build/mem_latency --huge | tee build/results/mem_latency_thp.csv +./build/page_fault_cost 512 | tee build/results/page_fault_cost.csv +./build/wakeup_latency 200000 | tee build/results/wakeup_latency.txt +./build/false_sharing 4 | tee build/results/false_sharing.txt diff --git a/scripts/setup_hugepages.sh b/scripts/setup_hugepages.sh index 627ad90..8fb9eba 100755 --- a/scripts/setup_hugepages.sh +++ b/scripts/setup_hugepages.sh @@ -1,10 +1,23 @@ #!/usr/bin/env bash +# Reserve hugetlb pages, optionally on one NUMA node, and show the result. +# sudo scripts/setup_hugepages.sh 1024 # 1024 x 2 MiB, system-wide +# sudo scripts/setup_hugepages.sh 512 1 # 512 x 2 MiB on node 1 +# Reserving at runtime can fall short on a fragmented system; for guaranteed +# pools use the kernel command line (hugepages=N, or hugepagesz=1G hugepages=N). set -euo pipefail - pages=${1:-128} +node=${2:-} if [ "$(id -u)" -ne 0 ]; then echo "Run as root to configure huge pages." >&2 exit 1 fi -echo "$pages" > /proc/sys/vm/nr_hugepages -echo "Configured $pages huge pages." +if [ -n "$node" ]; then + target=/sys/devices/system/node/node${node}/hugepages/hugepages-2048kB/nr_hugepages +else + target=/proc/sys/vm/nr_hugepages +fi +echo "$pages" > "$target" +got=$(cat "$target") +echo "requested $pages, got $got (2 MiB pages) via $target" +[ "$got" -lt "$pages" ] && echo "warning: memory too fragmented for the full reservation; try after boot or compact first" >&2 +grep -E '^HugePages_|^Hugepagesize' /proc/meminfo diff --git a/src/cache_analysis.py b/src/cache_analysis.py deleted file mode 100644 index 95d1b84..0000000 --- a/src/cache_analysis.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Cache-derived metrics.""" - -from __future__ import annotations - - -def cache_metrics(perf: dict[str, float]) -> dict[str, float]: - references = perf.get("cache-references", 0.0) - misses = perf.get("cache-misses", 0.0) - instructions = perf.get("instructions", 0.0) - return { - "cache_references": references, - "cache_misses": misses, - "cache_miss_rate_percent": (100 * misses / references) if references else 0.0, - "instructions_per_cache_miss": (instructions / misses) if misses else 0.0, - } diff --git a/src/hugepages.py b/src/hugepages.py deleted file mode 100644 index 0892e28..0000000 --- a/src/hugepages.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Transparent huge-page state reader.""" - -from pathlib import Path - - -def transparent_hugepages_enabled(path: str = "/sys/kernel/mm/transparent_hugepage/enabled") -> str | None: - try: - text = Path(path).read_text(encoding="utf-8") - except OSError: - return None - for option in text.split(): - if option.startswith("[") and option.endswith("]"): - return option[1:-1] - return None diff --git a/src/kptk/__init__.py b/src/kptk/__init__.py new file mode 100644 index 0000000..3c982d4 --- /dev/null +++ b/src/kptk/__init__.py @@ -0,0 +1,9 @@ +"""kptk: Linux kernel performance toolkit. + +Records raw /proc and /sys state with negligible overhead, reads hardware +counters through perf_event_open, and turns both into findings about CPU +scheduling, memory, NUMA locality, cache efficiency, page faults and huge pages. +Zero runtime dependencies: everything is Python standard library. +""" + +__version__ = "0.2.0" diff --git a/src/kptk/__main__.py b/src/kptk/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/kptk/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/kptk/analysis/__init__.py b/src/kptk/analysis/__init__.py new file mode 100644 index 0000000..7040641 --- /dev/null +++ b/src/kptk/analysis/__init__.py @@ -0,0 +1 @@ +"""Domain analyzers: cpu, memory, numa, cache, hugepages, and the engine that combines them.""" diff --git a/src/kptk/analysis/cache.py b/src/kptk/analysis/cache.py new file mode 100644 index 0000000..2c0157a --- /dev/null +++ b/src/kptk/analysis/cache.py @@ -0,0 +1,115 @@ +"""Cache, branch and TLB efficiency from hardware counters. + +Normalising by instructions (MPKI, misses per kilo-instruction) makes +numbers comparable across runs of different length, and connects directly +to cost: at ~80 ns per DRAM miss and 3 GHz, an LLC MPKI of 10 costs about 2.4 +cycles per instruction if nothing overlaps. + +The classification is a coarse "top-down lite". Real Top-down Microarchitecture +Analysis needs model-specific events (topdown-* slots); generic perf events +can only point in a direction. Findings say so. +""" + +from __future__ import annotations + +from ..topology import Topology +from .findings import DomainResult, Severity, Thresholds + + +def _ratio(a: float | None, b: float | None) -> float | None: + return a / b if a is not None and b else None + + +def analyze_cache(events: dict[str, dict[str, float]] | None, topo: Topology, th: Thresholds, anon_kib: int | None) -> DomainResult: + r = DomainResult() + m = r.metrics + m["caches"] = [{"level": c.level, "type": c.type, "size_kib": c.size_bytes // 1024, "shared_by_cpus": len(c.shared_cpus)} for c in topo.caches] + if not events: + m["available"] = False + r.add( + "cache.no_counters", + "cache", + Severity.INFO, + "No hardware counters in this recording; cache and TLB efficiency can't be measured", + {}, + "Use `kptk profile -- ` with perf_event_paranoid <= 2 (or CAP_PERFMON), or pass `--perf-stat` with saved `perf stat -x;` output.", + ) + return r + m["available"] = True + + def v(name: str) -> float | None: + e = events.get(name) + return e["value"] if e else None + + instr = v("instructions") + kinstr = instr / 1000 if instr else None + m["ipc"] = _ratio(instr, v("cycles")) + m["llc_miss_ratio"] = _ratio(v("cache-misses"), v("cache-references")) + m["llc_mpki"] = _ratio(v("cache-misses"), kinstr) + m["branch_miss_ratio"] = _ratio(v("branch-misses"), v("branches")) + m["branch_mpki"] = _ratio(v("branch-misses"), kinstr) + m["l1d_miss_ratio"] = _ratio(v("L1-dcache-load-misses"), v("L1-dcache-loads")) + m["dtlb_miss_ratio"] = _ratio(v("dTLB-load-misses"), v("dTLB-loads")) + m["dtlb_mpki"] = _ratio(v("dTLB-load-misses"), kinstr) + m["backend_stall_frac"] = _ratio(v("stalled-cycles-backend"), v("cycles")) + m["frontend_stall_frac"] = _ratio(v("stalled-cycles-frontend"), v("cycles")) + running = [e.get("running_frac", 1.0) for e in events.values() if e.get("running_frac") is not None] + m["min_running_frac"] = min(running) if running else None + + if m["min_running_frac"] is not None and m["min_running_frac"] < th.multiplex_running_warn: + r.add( + "cache.multiplexed", + "cache", + Severity.INFO, + f"Counters were multiplexed (some counted only {m['min_running_frac']:.0%} of the time); values are extrapolated", + {"min_running_frac": round(m["min_running_frac"], 3)}, + "Request fewer events per run, or group related events, for exact ratios. Treat cross-event ratios as estimates.", + "With more events than PMU counters the kernel time-slices them and scales by enabled/running time.", + ) + + ipc, mpki = m["ipc"], m["llc_mpki"] + memory_bound = (ipc is not None and ipc < th.ipc_low and mpki is not None and mpki > th.llc_mpki_warn) or (m["backend_stall_frac"] or 0) > 0.4 + if memory_bound: + llc = topo.llc + ws = f"; anonymous working set {anon_kib // 1024} MiB vs {llc.size_bytes >> 20} MiB L{llc.level}" if anon_kib and llc else "" + r.add( + "cache.memory_bound", + "cache", + Severity.WARNING, + f"Likely memory-bound: IPC {ipc:.2f}, LLC {mpki:.1f} misses per 1k instructions{ws}" + if ipc is not None and mpki is not None + else "Likely memory-bound (high backend stalls)", + {k: round(m[k], 4) for k in ("ipc", "llc_mpki", "llc_miss_ratio", "backend_stall_frac") if m.get(k) is not None}, + "Improve locality: tile or block loops to fit the LLC, use struct-of-arrays for scanned fields, prefer contiguous containers over pointer-chasing, and prefetch for irregular access.", + "Low IPC together with many LLC misses means the core is waiting on DRAM. Top-down analysis (perf stat --topdown) can confirm it on supported CPUs.", + ) + if m["branch_miss_ratio"] is not None and m["branch_miss_ratio"] > th.branch_miss_warn: + r.add( + "cache.bad_speculation", + "cache", + Severity.INFO, + f"Branch misprediction rate {m['branch_miss_ratio']:.1%} ({m['branch_mpki'] or 0:.1f} per 1k instructions)", + {"branch_miss_ratio": round(m["branch_miss_ratio"], 4)}, + "Make data-dependent branches predictable (sort, partition), use branchless selects, or use PGO so the compiler lays out hot paths.", + "Each mispredict flushes the pipeline (~15-20 cycles on modern cores).", + ) + if m["dtlb_miss_ratio"] is not None and m["dtlb_miss_ratio"] > th.dtlb_miss_warn: + r.add( + "cache.tlb_pressure", + "cache", + Severity.WARNING, + f"dTLB miss rate {m['dtlb_miss_ratio']:.1%} ({m['dtlb_mpki'] or 0:.2f} per 1k instructions): page walks are significant", + {"dtlb_miss_ratio": round(m["dtlb_miss_ratio"], 4)}, + "Back the hot heap with huge pages (madvise(MADV_HUGEPAGE), THP, or hugetlbfs). A 2 MiB page covers 512x the reach of a 4 KiB TLB entry. See the hugepages findings.", + "With a few thousand TLB entries x 4 KiB, reach is only a few MiB; larger random-access working sets miss constantly.", + ) + if ipc is not None and ipc > 2.0 and not memory_bound: + r.add( + "cache.efficient", + "cache", + Severity.INFO, + f"IPC {ipc:.2f}: the core pipeline is well utilised", + {"ipc": round(ipc, 3)}, + "Gains now come from doing less work (algorithms, vectorisation), not from memory layout.", + ) + return r diff --git a/src/kptk/analysis/cpu.py b/src/kptk/analysis/cpu.py new file mode 100644 index 0000000..5511f5a --- /dev/null +++ b/src/kptk/analysis/cpu.py @@ -0,0 +1,179 @@ +"""CPU utilisation and scheduler latency. + +Utilisation (/proc/stat) answers "how busy"; it can't distinguish a CPU +that's 100% busy with an empty run queue from one with ten tasks queued +behind it. Saturation is visible in three other places: + +* ``/proc/schedstat`` ``run_delay``: total ns tasks spent runnable but + waiting, per CPU (needs CONFIG_SCHEDSTATS, and ``kernel.sched_schedstats=1`` on + some kernels). +* ``/proc//schedstat``: the same, for one task. +* PSI ``/proc/pressure/cpu``: share of wall time at least one task waited. + +For a single process, *involuntary* context switches (preempted while still +runnable) point at CPU contention, and *voluntary* switches (blocked on +I/O, a futex or sleep) point at blocking. +""" + +from __future__ import annotations + +from collections import Counter + +from ..parsers.proc import busy_ticks, parse_pid_stat, parse_pid_status, parse_pressure, parse_proc_stat, parse_schedstat, parse_task_schedstat, total_ticks +from ..topology import Topology +from .findings import DomainResult, Severity, Thresholds +from .window import Window + + +def analyze_cpu(w: Window, topo: Topology, th: Thresholds) -> DomainResult: + r = DomainResult() + m = r.metrics + stat = w.pair("/proc/stat", parse_proc_stat) + if stat: + a, b, dt = stat + tot = {k: b.cpu_total[k] - a.cpu_total.get(k, 0) for k in b.cpu_total} + total = total_ticks(tot) or 1 + m["utilization"] = busy_ticks(tot) / total + for k in ("user", "system", "iowait", "steal", "softirq", "irq"): + m[f"{k}_frac"] = tot.get(k, 0) / total + per_cpu = {} + for cpu, times in b.cpus.items(): + d = {k: times[k] - a.cpus.get(cpu, {}).get(k, 0) for k in times} + per_cpu[cpu] = busy_ticks(d) / (total_ticks(d) or 1) + m["per_cpu_utilization"] = {c: round(u, 3) for c, u in sorted(per_cpu.items())} + m["context_switches_per_s"] = (b.ctxt - a.ctxt) / dt + running = [s.procs_running for _, s in w.series("/proc/stat", parse_proc_stat)] + m["procs_running_mean"] = sum(running) / len(running) + m["procs_running_per_cpu"] = m["procs_running_mean"] / max(1, len(topo.cpus)) + if per_cpu: + hot = max(per_cpu.values()) + spread = hot - min(per_cpu.values()) + m["per_cpu_spread"] = spread + if hot > 0.9 and spread > th.cpu_imbalance_warn: + busiest = sorted(per_cpu, key=lambda c: -per_cpu[c])[:4] + r.add( + "cpu.imbalance", + "cpu", + Severity.WARNING, + f"CPU load is imbalanced: CPU {busiest[0]} at {hot:.0%} while the least busy CPU is at {hot - spread:.0%}", + {"busiest_cpus": busiest, "spread": round(spread, 2)}, + "Check affinity masks (taskset/cgroup cpusets), IRQ affinity (/proc/interrupts, irqbalance) and single-threaded bottlenecks.", + "A pinned task or an IRQ storm can saturate one CPU while the scheduler can't move the work.", + ) + if m["steal_frac"] > th.steal_warn: + r.add( + "cpu.steal", + "cpu", + Severity.WARNING, + f"Hypervisor steal time is {m['steal_frac']:.1%} of CPU time", + {"steal_frac": round(m["steal_frac"], 4)}, + "The host is oversubscribed: move to dedicated or larger instances, or ask the platform team about vCPU overcommit.", + "Steal is time the vCPU was runnable but the hypervisor ran someone else.", + ) + if m["iowait_frac"] > th.iowait_warn: + r.add( + "cpu.iowait", + "cpu", + Severity.INFO, + f"iowait is {m['iowait_frac']:.1%}: CPUs idle while I/O is outstanding", + {"iowait_frac": round(m["iowait_frac"], 4)}, + "iowait is idle time, not busy time. Look at I/O pressure (PSI io) and the device queue rather than at CPU.", + ) + + sched = w.pair("/proc/schedstat", lambda t: parse_schedstat(t)[1]) + if sched: + a2, b2, dt = sched + delay = sum(b2[c].run_delay_ns - a2[c].run_delay_ns for c in b2 if c in a2) + slices = sum(b2[c].timeslices - a2[c].timeslices for c in b2 if c in a2) + # ns of waiting per ns of wall time = average number of tasks waiting in run queues + m["runq_waiting_tasks_avg"] = delay / 1e9 / dt + m["runq_wait_per_timeslice_ms"] = delay / slices / 1e6 if slices else 0.0 + per_cpu_wait = {c: (b2[c].run_delay_ns - a2[c].run_delay_ns) / 1e9 / dt for c in b2 if c in a2} + m["runq_waiting_tasks_per_cpu"] = {c: round(v, 3) for c, v in sorted(per_cpu_wait.items())} + else: + m["runq_waiting_tasks_avg"] = None + + psi = w.pair("/proc/pressure/cpu", parse_pressure) + if psi: + a3, b3, dt = psi + m["psi_cpu_some"] = (b3["some"].total_us - a3["some"].total_us) / 1e6 / dt + if m["psi_cpu_some"] > th.cpu_psi_some_warn: + r.add( + "cpu.saturation", + "cpu", + Severity.WARNING if m["psi_cpu_some"] < 0.5 else Severity.CRITICAL, + f"CPU saturation: some task was waiting for a CPU {m['psi_cpu_some']:.0%} of the time", + { + "psi_cpu_some": round(m["psi_cpu_some"], 3), + "runq_waiting_tasks_avg": round(m.get("runq_waiting_tasks_avg") or 0, 2), + "runq_wait_per_timeslice_ms": round(m.get("runq_wait_per_timeslice_ms") or 0, 3), + }, + "Reduce runnable threads (pool sizes), spread load across more CPUs, or isolate latency-critical threads (cpusets / isolcpus).", + "PSI counts wall time with at least one runnable task not running; it is saturation, not utilisation.", + ) + + _process(w, topo, th, r) + return r + + +def _process(w: Window, topo: Topology, th: Thresholds, r: DomainResult) -> None: + pid = w.rec.pid + if not pid: + return + m = r.metrics + ps = w.pair(f"/proc/{pid}/stat", parse_pid_stat) + if ps: + a, b, dt = ps + m["process_comm"] = b.comm + m["process_threads"] = b.num_threads + m["process_cpus_used"] = (b.utime + b.stime - a.utime - a.stime) / w.user_hz / dt + m["process_system_frac"] = (b.stime - a.stime) / max(1, b.utime + b.stime - a.utime - a.stime) + cpus_seen = Counter(s.processor for _, s in w.series(f"/proc/{pid}/stat", parse_pid_stat)) + nodes_seen = Counter(topo.cpu_node.get(c, 0) for c in cpus_seen.elements()) + m["process_cpus_seen"] = sorted(cpus_seen) + m["process_node_residency"] = {n: round(c / sum(nodes_seen.values()), 3) for n, c in sorted(nodes_seen.items())} + st = w.pair(f"/proc/{pid}/schedstat", parse_task_schedstat) + if st: + a2, b2, _ = st + on, wait, slices = b2.on_cpu_ns - a2.on_cpu_ns, b2.wait_ns - a2.wait_ns, b2.timeslices - a2.timeslices + share = wait / (wait + on) if wait + on else 0.0 + m["process_runq_wait_share"] = share + m["process_runq_wait_per_slice_ms"] = wait / slices / 1e6 if slices else 0.0 + if share > th.runq_wait_share_warn: + r.add( + "cpu.process_runq_wait", + "cpu", + Severity.WARNING if share < 0.3 else Severity.CRITICAL, + f"The process spent {share:.0%} of its runnable time waiting in a run queue", + {"wait_share": round(share, 3), "avg_wait_per_slice_ms": round(m["process_runq_wait_per_slice_ms"], 3)}, + "Give the process more CPUs or fewer competing threads; for latency-critical work, pin it to isolated cores.", + "Scheduler latency adds directly to request latency: the work was ready, but no CPU was free.", + ) + status = w.pair(f"/proc/{pid}/status", parse_pid_status) + if status: + a3, b3, dt = status + vol = (int(b3.get("voluntary_ctxt_switches", 0)) - int(a3.get("voluntary_ctxt_switches", 0))) / dt + invol = (int(b3.get("nonvoluntary_ctxt_switches", 0)) - int(a3.get("nonvoluntary_ctxt_switches", 0))) / dt + m["process_voluntary_switches_per_s"] = vol + m["process_involuntary_switches_per_s"] = invol + m["process_cpus_allowed"] = b3.get("Cpus_allowed_list") + if invol > th.involuntary_switch_rate_warn and invol > vol: + r.add( + "cpu.preemption", + "cpu", + Severity.WARNING, + f"{invol:,.0f} involuntary context switches/s: the process is being preempted", + {"involuntary_per_s": round(invol), "voluntary_per_s": round(vol)}, + "Too many runnable threads per CPU. Size thread pools to the CPU count, or isolate the hot threads.", + "Involuntary switches happen when a runnable task is descheduled because its slice expired or a higher-priority task woke.", + ) + elif vol > th.voluntary_switch_rate_warn: + r.add( + "cpu.blocking", + "cpu", + Severity.INFO, + f"{vol:,.0f} voluntary context switches/s: threads block and wake frequently", + {"voluntary_per_s": round(vol), "involuntary_per_s": round(invol)}, + "Look for lock contention (futex), small synchronous I/O or chatty producer/consumer queues; batch work or spin briefly before blocking.", + "Each block/wake pair costs a few microseconds of scheduler and cache-refill overhead.", + ) diff --git a/src/kptk/analysis/engine.py b/src/kptk/analysis/engine.py new file mode 100644 index 0000000..24ac441 --- /dev/null +++ b/src/kptk/analysis/engine.py @@ -0,0 +1,191 @@ +"""Run every domain analyzer over a recording and assemble the report.""" + +from __future__ import annotations + +import time +from typing import Any + +from ..parsers.proc import busy_ticks, parse_pressure, parse_proc_stat, parse_schedstat, parse_vmstat, total_ticks +from ..parsers.tools import PerfStatEvent +from ..record import Recording +from ..topology import Topology, from_static +from .cache import analyze_cache +from .cpu import analyze_cpu +from .findings import DomainResult, Finding, Severity, Thresholds +from .hugepages import analyze_hugepages +from .memory import analyze_memory +from .numa import analyze_numa +from .window import Window + + +def events_from_perf_stat(parsed: dict[str, PerfStatEvent]) -> dict[str, dict[str, float]]: + return {k: {"value": e.value, "running_frac": e.running_pct / 100} for k, e in parsed.items()} + + +def audit_config(static: dict[str, str], topo: Topology) -> DomainResult: + """Static configuration checks that need no workload.""" + r = DomainResult() + m = r.metrics + + def val(path: str) -> str | None: + v = static.get(path) + return v.strip() if v is not None else None + + m["swappiness"] = val("/proc/sys/vm/swappiness") + m["zone_reclaim_mode"] = val("/proc/sys/vm/zone_reclaim_mode") + m["numa_balancing"] = val("/proc/sys/kernel/numa_balancing") + m["perf_event_paranoid"] = val("/proc/sys/kernel/perf_event_paranoid") + m["governors"] = sorted(set(topo.governors.values())) + if m["zone_reclaim_mode"] not in (None, "0") and len(topo.nodes) > 1: + r.add( + "config.zone_reclaim", + "config", + Severity.WARNING, + f"vm.zone_reclaim_mode={m['zone_reclaim_mode']}: the kernel reclaims local page cache instead of allocating remotely", + {"zone_reclaim_mode": m["zone_reclaim_mode"]}, + "Good for strictly partitioned HPC workloads; usually bad for file servers and databases (page cache thrash). Default is 0.", + ) + if "powersave" in m["governors"]: + r.add( + "config.governor", + "config", + Severity.INFO, + "CPU frequency governor 'powersave' is active", + {"governors": m["governors"]}, + "On intel_pstate/amd-pstate 'powersave' is dynamic and often fine; for latency-critical services compare against 'performance' and check C-state exit latency.", + ) + try: + paranoid = int(m["perf_event_paranoid"] or 0) + except ValueError: + paranoid = 0 + if paranoid > 2: + r.add( + "config.perf_paranoid", + "config", + Severity.INFO, + f"kernel.perf_event_paranoid={paranoid}: unprivileged hardware counters are disabled", + {"perf_event_paranoid": paranoid}, + "For profiling sessions grant CAP_PERFMON to the profiler, or temporarily set perf_event_paranoid=2 (own processes, user+kernel).", + ) + return r + + +def use_summary(metrics: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + """Brendan Gregg's USE method: Utilisation, Saturation, Errors, per resource.""" + cpu, mem, numa, hp = metrics.get("cpu", {}), metrics.get("memory", {}), metrics.get("numa", {}), metrics.get("hugepages", {}) + avail = mem.get("mem_available_min_frac") + return [ + { + "resource": "CPU", + "utilization": cpu.get("utilization"), + "saturation": {"psi_some": cpu.get("psi_cpu_some"), "runq_waiting_tasks": cpu.get("runq_waiting_tasks_avg")}, + "errors": {"steal_frac": cpu.get("steal_frac")}, + }, + { + "resource": "Memory capacity", + "utilization": (1 - avail) if avail is not None else None, + "saturation": { + "psi_some": mem.get("psi_memory_some"), + "direct_reclaim_per_s": mem.get("direct_reclaim_stalls_per_s"), + "swap_pages_per_s": mem.get("swap_pages_per_s"), + }, + "errors": {"oom_kills": mem.get("oom_kills")}, + }, + { + "resource": "NUMA interconnect", + "utilization": numa.get("remote_alloc_frac"), + "saturation": {"pages_migrated_per_s": numa.get("pages_migrated_per_s")}, + "errors": {"policy_miss_frac": numa.get("policy_miss_frac")}, + }, + { + "resource": "Huge pages", + "utilization": hp.get("process_thp_coverage"), + "saturation": {"compact_stall_per_s": hp.get("compact_stall_per_s")}, + "errors": {"thp_fallback_frac": hp.get("thp_fallback_frac")}, + }, + ] + + +def timeseries(rec: Recording) -> dict[str, list[float | None]]: + """Per-interval rates for the report's sparklines.""" + out: dict[str, list[float | None]] = {"t": [], "cpu_util": [], "runq_waiting": [], "faults_per_s": [], "major_faults_per_s": [], "psi_mem_some": []} + for a, b in zip(rec.samples, rec.samples[1:], strict=False): + dt = b.ts - a.ts + if dt <= 0: + continue + out["t"].append(round(b.ts - rec.samples[0].ts, 3)) + util = rq = faults = majf = psim = None + if "/proc/stat" in a.files and "/proc/stat" in b.files: + sa, sb = parse_proc_stat(a.files["/proc/stat"]), parse_proc_stat(b.files["/proc/stat"]) + d = {k: sb.cpu_total[k] - sa.cpu_total.get(k, 0) for k in sb.cpu_total} + util = busy_ticks(d) / (total_ticks(d) or 1) + if "/proc/schedstat" in a.files and "/proc/schedstat" in b.files: + qa, qb = parse_schedstat(a.files["/proc/schedstat"])[1], parse_schedstat(b.files["/proc/schedstat"])[1] + rq = sum(qb[c].run_delay_ns - qa[c].run_delay_ns for c in qb if c in qa) / 1e9 / dt + if "/proc/vmstat" in a.files and "/proc/vmstat" in b.files: + va, vb = parse_vmstat(a.files["/proc/vmstat"]), parse_vmstat(b.files["/proc/vmstat"]) + faults = (vb.get("pgfault", 0) - va.get("pgfault", 0)) / dt + majf = (vb.get("pgmajfault", 0) - va.get("pgmajfault", 0)) / dt + if "/proc/pressure/memory" in a.files and "/proc/pressure/memory" in b.files: + pa, pb = parse_pressure(a.files["/proc/pressure/memory"]), parse_pressure(b.files["/proc/pressure/memory"]) + psim = (pb["some"].total_us - pa["some"].total_us) / 1e6 / dt + for k, v in (("cpu_util", util), ("runq_waiting", rq), ("faults_per_s", faults), ("major_faults_per_s", majf), ("psi_mem_some", psim)): + out[k].append(None if v is None else round(v, 4)) + return out + + +def data_quality(rec: Recording) -> dict[str, Any]: + pid = rec.pid + costs = [s.cost_s for s in rec.samples] + dq: dict[str, Any] = { + "samples": len(rec.samples), + "duration_s": round(rec.duration_s, 3), + "mean_sample_cost_ms": round(1e3 * sum(costs) / len(costs), 3) if costs else None, + "overruns": rec.meta.get("overruns", 0), + "missing": [], + } + first = rec.samples[0].files if rec.samples else {} + for path, why in ( + ("/proc/schedstat", "CONFIG_SCHEDSTATS off: no run-queue latency"), + ("/proc/pressure/cpu", "PSI unavailable (CONFIG_PSI or psi=0): no saturation stall times"), + ): + if path not in first: + dq["missing"].append(why) + if pid: + alive = [s for s in rec.samples if f"/proc/{pid}/stat" in s.files] + dq["process_samples"] = len(alive) + if len(alive) < len(rec.samples): + dq["missing"].append(f"pid {pid} exited or became unreadable after {len(alive)} samples") + if not any(f"/proc/{pid}/numa_maps" in s.files for s in rec.samples): + dq["missing"].append("numa_maps unreadable (permissions or short run): no per-process placement") + return dq + + +def analyze(rec: Recording, events: dict[str, dict[str, float]] | None = None, thresholds: Thresholds | None = None) -> dict[str, Any]: + th = thresholds or Thresholds() + w = Window(rec) + topo = from_static(rec.static) + events = events or (rec.counters or {}).get("events") + + cpu = analyze_cpu(w, topo, th) + mem = analyze_memory(w, th) + numa = analyze_numa(w, topo, th, cpu.metrics.get("process_node_residency")) + cache = analyze_cache(events, topo, th, mem.metrics.get("process_anon_kib")) + hp = analyze_hugepages(w, th, cache.metrics.get("dtlb_miss_ratio")) + cfg = audit_config(rec.static, topo) + + domains: dict[str, DomainResult] = {"cpu": cpu, "memory": mem, "numa": numa, "cache": cache, "hugepages": hp, "config": cfg} + findings: list[Finding] = [f for d in domains.values() for f in d.findings] + findings.sort(key=lambda f: (-int(f.severity), f.domain)) + metrics = {name: d.metrics for name, d in domains.items()} + return { + "generated_at": time.time(), + "meta": rec.meta, + "topology": topo.summary(), + "metrics": metrics, + "use": use_summary(metrics), + "findings": [f.to_dict() for f in findings], + "counters": events, + "timeseries": timeseries(rec), + "data_quality": data_quality(rec), + } diff --git a/src/kptk/analysis/findings.py b/src/kptk/analysis/findings.py new file mode 100644 index 0000000..1727fbc --- /dev/null +++ b/src/kptk/analysis/findings.py @@ -0,0 +1,91 @@ +"""Finding model and tunable thresholds.""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field, fields +from enum import IntEnum +from pathlib import Path +from typing import Any + + +class Severity(IntEnum): + INFO = 0 + WARNING = 1 + CRITICAL = 2 + + +@dataclass +class Finding: + id: str + domain: str + severity: Severity + title: str + evidence: dict[str, Any] + recommendation: str + why: str = "" + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["severity"] = self.severity.name.lower() + return d + + +@dataclass +class Thresholds: + """Defaults are conservative starting points. Override per fleet with ``--thresholds file.json``.""" + + # CPU / scheduler + cpu_psi_some_warn: float = 0.10 # share of time some task waited for CPU + runq_wait_share_warn: float = 0.10 # process time runnable-but-waiting / (that + on-CPU) + runq_wait_per_slice_ms_warn: float = 1.0 + involuntary_switch_rate_warn: float = 1000.0 # per second, for the target process + voluntary_switch_rate_warn: float = 20000.0 + steal_warn: float = 0.05 + iowait_warn: float = 0.10 + cpu_imbalance_warn: float = 0.50 # max - min per-CPU utilisation while some CPU is > 90% busy + # Memory + major_fault_rate_warn: float = 10.0 # per second + minor_fault_rate_warn: float = 200_000.0 + direct_reclaim_rate_warn: float = 1.0 # allocstall per second + swap_rate_warn: float = 10.0 # pages per second in+out + mem_psi_full_warn: float = 0.01 + mem_available_frac_warn: float = 0.05 + # NUMA + remote_alloc_warn: float = 0.10 + numa_placement_mismatch_warn: float = 0.30 # total variation distance between CPU-node and memory-node distributions + numa_hint_local_warn: float = 0.70 + node_free_frac_warn: float = 0.03 + numa_migrate_rate_warn: float = 10_000.0 # pages per second + # Cache / TLB + ipc_low: float = 1.0 + llc_mpki_warn: float = 5.0 + branch_miss_warn: float = 0.02 + dtlb_miss_warn: float = 0.01 + multiplex_running_warn: float = 0.95 + # Huge pages + thp_fallback_warn: float = 0.20 + thp_coverage_low: float = 0.20 + thp_min_anon_kib: int = 1 << 20 # only judge THP coverage for processes with >= 1 GiB anonymous memory + compact_stall_rate_warn: float = 1.0 + hugetlb_waste_kib_warn: int = 1 << 20 + + @classmethod + def load(cls, path: str | Path | None) -> Thresholds: + if not path: + return cls() + values = json.loads(Path(path).read_text()) + known = {f.name for f in fields(cls)} + unknown = set(values) - known + if unknown: + raise ValueError(f"unknown thresholds: {sorted(unknown)}") + return cls(**values) + + +@dataclass +class DomainResult: + metrics: dict[str, Any] = field(default_factory=dict) + findings: list[Finding] = field(default_factory=list) + + def add(self, *args: Any, **kwargs: Any) -> None: + self.findings.append(Finding(*args, **kwargs)) diff --git a/src/kptk/analysis/hugepages.py b/src/kptk/analysis/hugepages.py new file mode 100644 index 0000000..4b44201 --- /dev/null +++ b/src/kptk/analysis/hugepages.py @@ -0,0 +1,132 @@ +"""Transparent Huge Pages and hugetlbfs: configuration, effectiveness and cost. + +Huge pages trade TLB reach for allocation cost: + +* **Benefit**: one 2 MiB TLB entry covers 512 base pages, so random-access workloads + with big heaps spend far fewer cycles in page walks. +* **Costs**: obtaining contiguous 2 MiB physical blocks may need *compaction*. With + ``defrag=always`` that happens synchronously in the page-fault path, causing + latency spikes; with ``defer`` khugepaged does it in the background. Huge pages + can also bloat RSS (sparse use of a 2 MiB region) and must be split under pressure. + +The useful questions are: is the workload actually getting huge pages +(coverage), are allocations failing (fallback), and is anyone paying for +compaction (``compact_stall``)? +""" + +from __future__ import annotations + +import re + +from ..parsers.proc import parse_meminfo, parse_smaps_rollup, parse_vmstat +from ..parsers.sysfs import parse_bracket_choice +from .findings import DomainResult, Severity, Thresholds +from .window import Window, delta_dict + +_POOL = re.compile(r"/sys/kernel/mm/hugepages/hugepages-(\d+)kB/(\w+)$") +THP = "/sys/kernel/mm/transparent_hugepage/" + + +def hugetlb_pools(static: dict[str, str]) -> dict[int, dict[str, int]]: + pools: dict[int, dict[str, int]] = {} + for path, text in static.items(): + if m := _POOL.match(path): + pools.setdefault(int(m.group(1)), {})[m.group(2)] = int(text.strip() or 0) + return pools + + +def analyze_hugepages(w: Window, th: Thresholds, dtlb_miss_ratio: float | None) -> DomainResult: + r = DomainResult() + m = r.metrics + s = w.rec.static + enabled = parse_bracket_choice(s.get(THP + "enabled", "")) + defrag = parse_bracket_choice(s.get(THP + "defrag", "")) + m["thp_enabled"], m["thp_defrag"] = enabled, defrag + m["khugepaged_max_ptes_none"] = (s.get(THP + "khugepaged/max_ptes_none") or "").strip() or None + + if enabled == "always" and defrag == "always": + r.add( + "hp.sync_compaction", + "hugepages", + Severity.WARNING, + "THP enabled=always with defrag=always: page faults may stall on synchronous compaction", + {"enabled": enabled, "defrag": defrag}, + "Use defrag=defer+madvise: background compaction for everyone, synchronous only for regions that asked with MADV_HUGEPAGE.", + "Under fragmentation every anonymous fault can enter direct compaction, a classic source of multi-ms latency spikes in databases and JVMs.", + ) + + vm = w.pair("/proc/vmstat", parse_vmstat) + if vm: + d = delta_dict(vm[0], vm[1]) + dt = vm[2] + alloc, fallback = d.get("thp_fault_alloc", 0), d.get("thp_fault_fallback", 0) + m["thp_fault_alloc_per_s"] = alloc / dt + m["thp_fallback_frac"] = fallback / (alloc + fallback) if alloc + fallback else None + m["thp_collapse_per_s"] = d.get("thp_collapse_alloc", 0) / dt + m["thp_split_per_s"] = d.get("thp_split_page", 0) / dt + m["compact_stall_per_s"] = d.get("compact_stall", 0) / dt + m["compact_fail_frac"] = d.get("compact_fail", 0) / d["compact_stall"] if d.get("compact_stall") else None + if m["thp_fallback_frac"] is not None and m["thp_fallback_frac"] > th.thp_fallback_warn and alloc + fallback >= 10: + r.add( + "hp.thp_fallback", + "hugepages", + Severity.WARNING, + f"{m['thp_fallback_frac']:.0%} of huge-page faults fell back to 4 KiB pages", + {"thp_fault_alloc": alloc, "thp_fault_fallback": fallback}, + "Physical memory is fragmented. Reserve hugetlbfs pages at boot for critical workloads, raise vm.min_free_kbytes, or trigger compaction off-peak (echo 1 > /proc/sys/vm/compact_memory).", + ) + if m["compact_stall_per_s"] > th.compact_stall_rate_warn: + r.add( + "hp.compaction_stalls", + "hugepages", + Severity.WARNING, + f"{m['compact_stall_per_s']:.1f} direct compaction stalls/s", + {"compact_stall_per_s": round(m["compact_stall_per_s"], 2), "compact_fail_frac": m["compact_fail_frac"]}, + "Tasks are compacting memory synchronously to get contiguous pages. Prefer THP defrag=defer+madvise and keep free-memory headroom.", + ) + + meminfo = w.latest("/proc/meminfo", parse_meminfo) + pools = hugetlb_pools(s) + m["hugetlb_pools"] = pools + for size_kib, p in pools.items(): + nr, free, resv = p.get("nr_hugepages", 0), p.get("free_hugepages", 0), p.get("resv_hugepages", 0) + idle_kib = max(0, free - resv) * size_kib + if nr and idle_kib >= th.hugetlb_waste_kib_warn: + r.add( + "hp.hugetlb_idle", + "hugepages", + Severity.INFO, + f"{idle_kib >> 20} GiB of {size_kib // 1024} MiB hugetlb pages are reserved in the pool but unused", + {"page_size_kib": size_kib, "nr": nr, "free": free, "reserved": resv}, + "hugetlb pages are unavailable to the rest of the system. Shrink vm.nr_hugepages, or confirm the consumer (DPDK, a DB) starts later.", + ) + if meminfo: + m["anon_huge_kib_system"] = meminfo.get("AnonHugePages", 0) + + pid = w.rec.pid + if pid: + smaps = w.latest(f"/proc/{pid}/smaps_rollup", parse_smaps_rollup) + if smaps: + anon = smaps.get("Anonymous", 0) + huge = smaps.get("AnonHugePages", 0) + m["process_anon_kib"] = anon + m["process_thp_coverage"] = huge / anon if anon else None + m["process_hugetlb_kib"] = smaps.get("Private_Hugetlb", 0) + smaps.get("Shared_Hugetlb", 0) + coverage = m["process_thp_coverage"] or 0.0 + if anon >= th.thp_min_anon_kib and coverage < th.thp_coverage_low and not m["process_hugetlb_kib"]: + tlb = f" and dTLB miss rate is {dtlb_miss_ratio:.1%}" if dtlb_miss_ratio is not None else "" + sev = Severity.WARNING if (dtlb_miss_ratio or 0) > th.dtlb_miss_warn else Severity.INFO + hint = ( + "madvise(MADV_HUGEPAGE) on the hot heap regions (THP is in madvise mode)" + if enabled == "madvise" + else "enabling THP (or hugetlbfs for predictable latency)" + ) + r.add( + "hp.low_coverage", + "hugepages", + sev, + f"Only {coverage:.0%} of the process's {anon >> 10} MiB anonymous memory is backed by huge pages{tlb}", + {"anon_kib": anon, "anon_huge_kib": huge, "thp_enabled": enabled}, + f"Try {hint}. For jemalloc/tcmalloc there are allocator options to request huge pages. Measure: the benefit depends on random-access TLB pressure.", + ) + return r diff --git a/src/kptk/analysis/memory.py b/src/kptk/analysis/memory.py new file mode 100644 index 0000000..064ac3d --- /dev/null +++ b/src/kptk/analysis/memory.py @@ -0,0 +1,128 @@ +"""Memory pressure and page faults. + +* **Minor fault**: the page is in memory (or just needs zeroing), so only a page-table + update is needed. Costs ~0.1-1 µs, but at millions per second it shows up as + system time. Typical sources are first-touch of fresh allocations, fork/COW, and mmap churn. +* **Major fault**: the page had to be read from storage (page cache miss, swap-in). + Costs ~10 µs (NVMe) to ~10 ms (HDD), and stalls the thread. +* **Direct reclaim** (``allocstall_*``, ``pgscan_direct``): an allocating task had + to free memory itself because kswapd fell behind. It turns allocation into a + latency spike. +""" + +from __future__ import annotations + +from ..parsers.proc import parse_meminfo, parse_pid_stat, parse_pressure, parse_smaps_rollup, parse_vmstat +from .findings import DomainResult, Severity, Thresholds +from .window import Window, delta_dict, rate + + +def analyze_memory(w: Window, th: Thresholds) -> DomainResult: + r = DomainResult() + m = r.metrics + vm = w.pair("/proc/vmstat", parse_vmstat) + if vm: + a, b, dt = vm + d = delta_dict(a, b) + allocstall = [k for k in d if k.startswith("allocstall")] + m["faults_per_s"] = rate(d, dt, "pgfault") + m["major_faults_per_s"] = rate(d, dt, "pgmajfault") + m["swap_pages_per_s"] = rate(d, dt, "pswpin", "pswpout") + m["direct_reclaim_stalls_per_s"] = rate(d, dt, *allocstall) + m["pgscan_direct_per_s"] = rate(d, dt, "pgscan_direct") + m["pgscan_kswapd_per_s"] = rate(d, dt, "pgscan_kswapd") + m["oom_kills"] = d.get("oom_kill", 0) + m["workingset_refault_per_s"] = rate(d, dt, "workingset_refault_file", "workingset_refault_anon") + if m["direct_reclaim_stalls_per_s"] > th.direct_reclaim_rate_warn: + r.add( + "mem.direct_reclaim", + "memory", + Severity.WARNING, + f"Direct reclaim: {m['direct_reclaim_stalls_per_s']:.1f} allocation stalls/s", + {"allocstall_per_s": round(m["direct_reclaim_stalls_per_s"], 2), "pgscan_direct_per_s": round(m["pgscan_direct_per_s"])}, + "Raise vm.min_free_kbytes / watermark_scale_factor so kswapd starts earlier, or reduce memory pressure (cgroup limits, page cache footprint).", + "In direct reclaim the allocating thread scans and frees pages itself, which adds ms-scale tail latency to malloc and page faults.", + ) + if m["swap_pages_per_s"] > th.swap_rate_warn: + r.add( + "mem.swapping", + "memory", + Severity.CRITICAL, + f"Swapping {m['swap_pages_per_s']:.0f} pages/s", + {"pswpin_out_per_s": round(m["swap_pages_per_s"])}, + "The working set doesn't fit. Reduce memory use, add memory, or set memory.high on low-priority cgroups.", + ) + if m["oom_kills"]: + r.add( + "mem.oom", + "memory", + Severity.CRITICAL, + f"{m['oom_kills']} OOM kill(s) during the window", + {"oom_kills": m["oom_kills"]}, + "Check dmesg for the victim and the cgroup limit that was hit.", + ) + + mi = w.series("/proc/meminfo", parse_meminfo) + if mi: + total = mi[-1][1].get("MemTotal", 0) + avail_min = min(x.get("MemAvailable", total) for _, x in mi) + m["mem_total_kib"] = total + m["mem_available_min_frac"] = avail_min / total if total else None + if total and avail_min / total < th.mem_available_frac_warn: + r.add( + "mem.low_available", + "memory", + Severity.WARNING, + f"MemAvailable dropped to {avail_min / total:.1%} of RAM", + {"mem_available_min_kib": avail_min}, + "Expect reclaim and refaults. Find the largest consumers (smaps_rollup, cgroup memory.stat).", + ) + + psi = w.pair("/proc/pressure/memory", parse_pressure) + if psi: + a2, b2, dt = psi + m["psi_memory_some"] = (b2["some"].total_us - a2["some"].total_us) / 1e6 / dt + m["psi_memory_full"] = (b2["full"].total_us - a2["full"].total_us) / 1e6 / dt + if m["psi_memory_full"] > th.mem_psi_full_warn: + r.add( + "mem.pressure", + "memory", + Severity.CRITICAL if m["psi_memory_full"] > 0.1 else Severity.WARNING, + f"Memory stalls: all non-idle tasks were blocked on memory {m['psi_memory_full']:.1%} of the time", + {"psi_memory_some": round(m["psi_memory_some"], 4), "psi_memory_full": round(m["psi_memory_full"], 4)}, + "This is lost throughput, not just latency. Reduce working set or memory limits pressure before tuning anything else.", + "PSI 'full' means no task could make progress because of reclaim, refaults or swap-in.", + ) + + pid = w.rec.pid + if pid: + ps = w.pair(f"/proc/{pid}/stat", parse_pid_stat) + if ps: + a3, b3, dt = ps + m["process_minor_faults_per_s"] = (b3.minflt - a3.minflt) / dt + m["process_major_faults_per_s"] = (b3.majflt - a3.majflt) / dt + m["process_rss_kib"] = b3.rss_pages * w.page_kib + if m["process_major_faults_per_s"] > th.major_fault_rate_warn: + r.add( + "mem.major_faults", + "memory", + Severity.WARNING, + f"{m['process_major_faults_per_s']:.0f} major page faults/s in the process", + {"major_faults_per_s": round(m["process_major_faults_per_s"], 1)}, + "Each major fault blocks on storage. Prefetch with madvise(MADV_WILLNEED) or readahead, mlock hot regions, or check whether this is swap-in.", + ) + if m["process_minor_faults_per_s"] > th.minor_fault_rate_warn: + r.add( + "mem.minor_fault_storm", + "memory", + Severity.WARNING, + f"{m['process_minor_faults_per_s']:,.0f} minor faults/s: heavy first-touch or mmap churn", + {"minor_faults_per_s": round(m["process_minor_faults_per_s"])}, + "Reuse buffers instead of freeing and reallocating (the allocator may return memory to the OS), pre-fault with MAP_POPULATE, or use huge pages to cut fault count by 512x.", + "Allocators like glibc malloc release large free blocks with munmap/madvise, so the next allocation faults again.", + ) + smaps = w.latest(f"/proc/{pid}/smaps_rollup", parse_smaps_rollup) + if smaps: + m["process_anon_kib"] = smaps.get("Anonymous", 0) + m["process_swap_kib"] = smaps.get("Swap", 0) + return r diff --git a/src/kptk/analysis/numa.py b/src/kptk/analysis/numa.py new file mode 100644 index 0000000..dc8a973 --- /dev/null +++ b/src/kptk/analysis/numa.py @@ -0,0 +1,183 @@ +"""NUMA locality. + +The per-node ``numastat`` counters are often misread: + +* ``numa_hit`` / ``numa_miss``: the allocation landed on the node the *memory + policy* preferred, or didn't (the preferred node was full). Policy success, + not locality. +* ``local_node`` / ``other_node``: the allocation landed on the node *where the + allocating task was running*, or on another one. This is allocation + locality. + +Neither says where memory is *accessed* from later, after the scheduler moves +threads. Two better sources are: + +* AutoNUMA hinting faults (``numa_hint_faults`` / ``numa_hint_faults_local`` in + /proc/vmstat): sampled real accesses, when ``kernel.numa_balancing=1``. +* ``/proc//numa_maps`` (where the pages live) compared with where the + process's threads actually ran (``/proc//stat`` processor field). Combined + with the SLIT distance matrix, that gives a relative memory-access cost. +""" + +from __future__ import annotations + +import re + +from ..parsers.proc import parse_meminfo, parse_numa_maps, parse_vmstat +from ..topology import Topology +from .findings import DomainResult, Severity, Thresholds +from .window import Window, delta_dict + +_NODE_FILE = re.compile(r"/sys/devices/system/node/node(\d+)/(numastat|meminfo)$") + + +def _numastat(text: str) -> dict[str, int]: + return {k: int(v) for k, v in (line.split() for line in text.splitlines() if line.strip())} + + +def analyze_numa(w: Window, topo: Topology, th: Thresholds, process_node_residency: dict[int, float] | None) -> DomainResult: + r = DomainResult() + m = r.metrics + nodes = topo.nodes + m["nodes"] = len(nodes) + m["distances"] = topo.distances + if len(nodes) < 2: + m["applicable"] = False + r.add( + "numa.single_node", + "numa", + Severity.INFO, + "Single NUMA node: locality analysis doesn't apply", + {"nodes": len(nodes)}, + "On multi-socket or sub-NUMA-clustered servers, record there to get locality findings.", + ) + return r + m["applicable"] = True + + per_node: dict[int, dict[str, int]] = {} + for n in nodes: + pair = w.pair(f"/sys/devices/system/node/node{n}/numastat", _numastat) + if pair: + per_node[n] = delta_dict(pair[0], pair[1]) + if per_node: + local = sum(d.get("local_node", 0) for d in per_node.values()) + other = sum(d.get("other_node", 0) for d in per_node.values()) + hit = sum(d.get("numa_hit", 0) for d in per_node.values()) + miss = sum(d.get("numa_miss", 0) for d in per_node.values()) + m["remote_alloc_frac"] = other / (local + other) if local + other else None + m["policy_miss_frac"] = miss / (hit + miss) if hit + miss else None + m["per_node_alloc"] = { + n: {"local": d.get("local_node", 0), "other": d.get("other_node", 0), "miss": d.get("numa_miss", 0)} for n, d in per_node.items() + } + if m["remote_alloc_frac"] and m["remote_alloc_frac"] > th.remote_alloc_warn: + r.add( + "numa.remote_alloc", + "numa", + Severity.WARNING, + f"{m['remote_alloc_frac']:.0%} of page allocations landed on a node other than the allocating CPU's", + {"remote_alloc_frac": round(m["remote_alloc_frac"], 3), "policy_miss_frac": round(m["policy_miss_frac"] or 0, 3)}, + "Bind CPU and memory together (numactl --cpunodebind=N --membind=N, or cpuset.mems) and check for interleave or preferred policies set by the runtime.", + "Remote DRAM costs roughly 1.3-2x local latency and shares inter-socket link bandwidth.", + ) + + vm = w.pair("/proc/vmstat", parse_vmstat) + balancing = (w.rec.static.get("/proc/sys/kernel/numa_balancing") or "").strip() + m["numa_balancing"] = balancing or None + if vm: + d = delta_dict(vm[0], vm[1]) + hints, local_hints = d.get("numa_hint_faults", 0), d.get("numa_hint_faults_local", 0) + m["hint_fault_local_frac"] = local_hints / hints if hints else None + m["pages_migrated_per_s"] = d.get("numa_pages_migrated", 0) / vm[2] + if hints and local_hints / hints < th.numa_hint_local_warn: + r.add( + "numa.access_locality", + "numa", + Severity.WARNING, + f"Only {local_hints / hints:.0%} of sampled memory accesses were node-local (AutoNUMA hinting faults)", + {"hint_faults": hints, "local": local_hints}, + "Threads and their data are on different nodes. Partition data per node, or pin threads to the node holding their memory.", + ) + if m["pages_migrated_per_s"] > th.numa_migrate_rate_warn: + r.add( + "numa.migration_storm", + "numa", + Severity.WARNING, + f"AutoNUMA is migrating {m['pages_migrated_per_s']:,.0f} pages/s", + {"pages_migrated_per_s": round(m["pages_migrated_per_s"])}, + "Threads are bouncing between nodes and dragging memory with them. Pin the workload, or disable numa_balancing for it.", + "Each migration copies 4 KiB (or 2 MiB) and causes TLB shootdowns; it only pays off if the thread then stays put.", + ) + if balancing == "0" and m.get("remote_alloc_frac") and m["remote_alloc_frac"] > th.remote_alloc_warn: + r.add( + "numa.balancing_off", + "numa", + Severity.INFO, + "kernel.numa_balancing is off while remote allocations are high", + {}, + "Either pin explicitly or enable AutoNUMA (sysctl kernel.numa_balancing=1) and measure.", + ) + + # Per-node free memory: one full node silently pushes allocations remote. + for n in nodes: + info = w.latest(f"/sys/devices/system/node/node{n}/meminfo", parse_meminfo) + if info and info.get("MemTotal"): + frac = info.get("MemFree", 0) / info["MemTotal"] + m.setdefault("node_free_frac", {})[n] = round(frac, 4) + if frac < th.node_free_frac_warn: + others = [x for x in nodes if x != n] + r.add( + "numa.node_exhausted", + "numa", + Severity.WARNING, + f"Node {n} is nearly full ({frac:.1%} free): new allocations will spill to node(s) {others} or trigger reclaim", + {"node": n, "free_frac": round(frac, 4), "zone_reclaim_mode": (w.rec.static.get("/proc/sys/vm/zone_reclaim_mode") or "").strip()}, + "Rebalance placement or interleave large shared buffers. With zone_reclaim_mode=0 (default) the kernel goes remote rather than reclaim locally.", + ) + + _process_placement(w, topo, th, r, process_node_residency) + return r + + +def _process_placement(w: Window, topo: Topology, th: Thresholds, r: DomainResult, residency: dict[int, float] | None) -> None: + pid = w.rec.pid + if not pid: + return + maps = w.latest(f"/proc/{pid}/numa_maps", parse_numa_maps) + if not maps or not maps.total_pages: + return + m = r.metrics + mem_frac = {n: c / maps.total_pages for n, c in sorted(maps.pages_by_node.items())} + m["process_memory_by_node"] = {n: round(f, 3) for n, f in mem_frac.items()} + m["process_memory_policies"] = maps.policies + if not residency: + return + # Expected distance a memory access travels, given where threads ran and where pages live; 10 = all local. + # Under a uniform-access assumption (every thread touches all of the heap): + cost = sum(cf * mf * topo.distance(cn, mn) for cn, cf in residency.items() for mn, mf in mem_frac.items()) + remote = sum(cf * mf for cn, cf in residency.items() for mn, mf in mem_frac.items() if cn != mn) + m["process_expected_distance"] = round(cost, 2) + m["process_remote_access_frac_est"] = round(remote, 3) + # A process spread over both nodes with memory spread the same way is fine + # (each thread can work on local data); 50% "remote" under uniform access is + # just the interleave baseline. What signals misplacement is a *mismatch* + # between where threads run and where memory lives: total variation distance. + nodes = set(residency) | set(mem_frac) + mismatch = 0.5 * sum(abs(residency.get(n, 0.0) - mem_frac.get(n, 0.0)) for n in nodes) + m["process_placement_mismatch"] = round(mismatch, 3) + if mismatch > th.numa_placement_mismatch_warn: + main_cpu_node = max(residency, key=lambda k: residency[k]) + main_mem_node = max(mem_frac, key=lambda k: mem_frac[k]) + r.add( + "numa.process_misplaced", + "numa", + Severity.WARNING if mismatch < 0.6 else Severity.CRITICAL, + f"About {remote:.0%} of the process's memory accesses are likely remote: threads ran mostly on node {main_cpu_node}, memory sits mostly on node {main_mem_node}", + { + "cpu_node_residency": residency, + "memory_by_node": m["process_memory_by_node"], + "placement_mismatch": m["process_placement_mismatch"], + "expected_slit_distance": m["process_expected_distance"], + }, + f"Start it with numactl --cpunodebind={main_mem_node} --membind={main_mem_node}, or migratepages {pid} {main_mem_node} {main_cpu_node}.", + "Typical cause: memory was first-touched by an initialisation thread on one node, then worker threads were scheduled on another.", + ) diff --git a/src/kptk/analysis/window.py b/src/kptk/analysis/window.py new file mode 100644 index 0000000..e3cb926 --- /dev/null +++ b/src/kptk/analysis/window.py @@ -0,0 +1,53 @@ +"""Delta helpers over a recording: first/last sample, per-file deltas, per-sample rates.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TypeVar + +from ..record import Recording, Sample + +T = TypeVar("T") + + +class Window: + """The analysis window: from the first to the last sample of a recording.""" + + def __init__(self, rec: Recording) -> None: + if len(rec.samples) < 2: + raise ValueError("a recording needs at least 2 samples to compute rates") + self.rec = rec + self.first, self.last = rec.samples[0], rec.samples[-1] + self.dt = self.last.ts - self.first.ts + self.user_hz = int(rec.meta.get("user_hz") or 100) + self.page_kib = int(rec.meta.get("page_size") or 4096) // 1024 + + def pair(self, path: str, parse: Callable[[str], T]) -> tuple[T, T, float] | None: + """Parsed values at the first and last sample containing ``path``, plus the elapsed time between them.""" + present = [s for s in self.rec.samples if path in s.files] + if len(present) < 2: + return None + a, b = present[0], present[-1] + return parse(a.files[path]), parse(b.files[path]), b.ts - a.ts + + def series(self, path: str, parse: Callable[[str], T]) -> list[tuple[Sample, T]]: + return [(s, parse(s.files[path])) for s in self.rec.samples if path in s.files] + + def latest(self, path: str, parse: Callable[[str], T]) -> T | None: + text = self.rec.latest(path) + return parse(text) if text is not None else None + + def has(self, path: str) -> bool: + return any(path in s.files for s in self.rec.samples) + + +def delta_dict(a: dict[str, int], b: dict[str, int]) -> dict[str, int]: + return {k: b[k] - a[k] for k in b if k in a} + + +def rate(d: dict[str, int], dt: float, *keys: str) -> float: + return sum(d.get(k, 0) for k in keys) / dt if dt > 0 else 0.0 + + +def ratio(num: float, den: float) -> float | None: + return num / den if den else None diff --git a/src/kptk/cli.py b/src/kptk/cli.py new file mode 100644 index 0000000..cfdfd4c --- /dev/null +++ b/src/kptk/cli.py @@ -0,0 +1,249 @@ +"""``kptk`` command-line interface (argparse, standard library only).""" + +from __future__ import annotations + +import argparse +import json +import signal +import sys +from pathlib import Path +from typing import Any + +from . import __version__ + + +def _write_outputs(report: dict[str, Any], args: argparse.Namespace) -> None: + from .report import render + + if getattr(args, "json", None): + Path(args.json).write_text(json.dumps(report, indent=2, default=str)) + text = render(report) + if getattr(args, "md", None): + Path(args.md).write_text(text) + print(f"report written to {args.md}", file=sys.stderr) + else: + print(text) + + +def _gate(report: dict[str, Any], fail_on: str | None) -> int: + """Exit status for CI: 3 if any finding is at or above ``fail_on``.""" + if not fail_on: + return 0 + order = {"info": 0, "warning": 1, "critical": 2} + worst = max((order[f["severity"]] for f in report["findings"]), default=-1) + return 3 if worst >= order[fail_on] else 0 + + +def cmd_inspect(args: argparse.Namespace) -> int: + from .analysis.engine import audit_config + from .analysis.hugepages import hugetlb_pools + from .parsers.sysfs import parse_bracket_choice + from .record import Recorder + from .topology import from_static + + static = Recorder().capture_static() + topo = from_static(static) + audit = audit_config(static, topo) + thp = "/sys/kernel/mm/transparent_hugepage/" + doc: dict[str, Any] = { + "topology": topo.summary(), + "config": audit.metrics, + "thp": {"enabled": parse_bracket_choice(static.get(thp + "enabled", "")), "defrag": parse_bracket_choice(static.get(thp + "defrag", ""))}, + "hugetlb_pools": hugetlb_pools(static), + "findings": [f.to_dict() for f in audit.findings], + } + if args.json: + print(json.dumps(doc, indent=2, default=str)) + return 0 + t = doc["topology"] + print(f"CPUs {t['cpus']} | cores {t['physical_cores']} | sockets {t['packages']} | SMT {'on' if t['smt'] else 'off'} | NUMA nodes {t['numa_nodes']}") + for node, cpus in t["node_cpus"].items(): + print(f" node{node}: cpus {cpus} distances {t['distances'].get(node, '-')}") + for c in t["caches"]: + print(f" L{c['level']} {c['type']:<12} {c['size_kib']:>7} KiB x{c['instances']} (shared by {c['shared_by_cpus']} CPUs)") + print(f"THP enabled={doc['thp']['enabled']} defrag={doc['thp']['defrag']} | hugetlb pools {doc['hugetlb_pools'] or 'none'}") + print(f"governors {t['governors'] or '-'} | " + " | ".join(f"{k}={v}" for k, v in doc["config"].items() if k != "governors")) + for f in doc["findings"]: + print(f"[{f['severity']}] {f['title']}\n -> {f['recommendation']}") + return 0 + + +def cmd_record(args: argparse.Namespace) -> int: + from .record import Recorder + + rec_ = Recorder(interval_s=args.interval, pid=args.pid, slow_every=args.slow_every) + signal.signal(signal.SIGTERM, lambda *_: rec_.stop_event.set()) + print(f"recording every {args.interval}s for {args.duration}s" + (f" (pid {args.pid})" if args.pid else "") + f" -> {args.output}", file=sys.stderr) + try: + rec = rec_.run(duration_s=args.duration) + except KeyboardInterrupt: + return 130 + rec.save(args.output) + print(f"{len(rec.samples)} samples, mean cost {1e3 * sum(s.cost_s for s in rec.samples) / len(rec.samples):.2f} ms/sample", file=sys.stderr) + return 0 + + +def cmd_analyze(args: argparse.Namespace) -> int: + from .analysis.engine import analyze, events_from_perf_stat + from .analysis.findings import Thresholds + from .parsers.tools import parse_perf_stat + from .record import Recording + + rec = Recording.load(args.recording) + events = events_from_perf_stat(parse_perf_stat(Path(args.perf_stat).read_text())) if args.perf_stat else None + report = analyze(rec, events, Thresholds.load(args.thresholds)) + _write_outputs(report, args) + return _gate(report, args.fail_on) + + +def cmd_profile(args: argparse.Namespace) -> int: + from .analysis.engine import analyze + from .analysis.findings import Thresholds + from .profile import profile + + cmd = args.command[1:] if args.command and args.command[0] == "--" else args.command + if not cmd: + print("usage: kptk profile [options] -- [args...]", file=sys.stderr) + return 2 + rec, rc = profile(cmd, args.interval, args.events.split(",") if args.events else None) + if args.output: + rec.save(args.output) + if rec.counters and rec.counters.get("error"): + print(f"note: hardware counters unavailable: {rec.counters['error']}", file=sys.stderr) + if len(rec.samples) < 2: + print("command finished before two samples were taken; lower --interval", file=sys.stderr) + return rc or 1 + report = analyze(rec, None, Thresholds.load(args.thresholds)) + _write_outputs(report, args) + return _gate(report, args.fail_on) or rc + + +def cmd_parse(args: argparse.Namespace) -> int: + """v0.1 workflow: analyse saved perf stat / vmstat / numastat text.""" + from .analysis.cache import analyze_cache + from .analysis.engine import events_from_perf_stat + from .analysis.findings import Thresholds + from .parsers.tools import parse_numastat_output, parse_perf_stat, parse_vmstat_output + from .topology import Topology + + perf = parse_perf_stat(Path(args.perf_file).read_text()) + events = events_from_perf_stat(perf) + cache = analyze_cache(events, Topology(), Thresholds(), None) + doc: dict[str, Any] = {"perf": {k: v.value for k, v in perf.items()}, "cache": cache.metrics, "findings": [f.to_dict() for f in cache.findings]} + sw = {k: perf[k].value for k in ("task-clock", "context-switches", "cpu-migrations", "minor-faults", "major-faults", "page-faults") if k in perf} + if sw: + doc["software"] = sw + if args.vmstat_file: + doc["vmstat"] = parse_vmstat_output(Path(args.vmstat_file).read_text()) + if args.numastat_file: + ns = parse_numastat_output(Path(args.numastat_file).read_text()) + doc["numastat"] = ns + local, other = ns.get("local_node", {}).get("total"), ns.get("other_node", {}).get("total") + if local is not None and other is not None and local + other: + doc["remote_alloc_frac"] = other / (local + other) + hit, miss = ns.get("numa_hit", {}).get("total"), ns.get("numa_miss", {}).get("total") + if hit is not None and miss is not None and hit + miss: + doc["policy_miss_frac"] = miss / (hit + miss) + text = json.dumps(doc, indent=2, default=str) + if args.output: + Path(args.output).write_text(text + "\n") + else: + print(text) + return 0 + + +def cmd_compare(args: argparse.Namespace) -> int: + from .compare import compare, to_markdown + + before = [json.loads(Path(p).read_text()) for p in args.before] + after = [json.loads(Path(p).read_text()) for p in args.after] + changes = compare(before, after, args.min_change, all_metrics=args.all) + print(to_markdown(changes, len(before), len(after))) + return 3 if args.fail_on_regression and any(c.verdict.startswith("regressed") for c in changes) else 0 + + +def cmd_demo(args: argparse.Namespace) -> int: + from .analysis.engine import analyze + from .synth import SCENARIOS, scenario + + if args.list: + for name, (desc, _) in SCENARIOS.items(): + print(f"{name:<16} {desc}") + return 0 + rec = scenario(args.scenario) + if args.save: + rec.save(args.save) + report = analyze(rec) + _write_outputs(report, args) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="kptk", description="Linux kernel performance toolkit: scheduling, memory, NUMA, cache, page faults, huge pages.") + p.add_argument("--version", action="version", version=f"kptk {__version__}") + sub = p.add_subparsers(dest="cmd", required=True) + + def outputs(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--json", help="write the full report as JSON") + sp.add_argument("--md", help="write the Markdown report here (default: stdout)") + sp.add_argument("--thresholds", help="JSON file overriding analysis thresholds") + sp.add_argument("--fail-on", choices=["info", "warning", "critical"], help="exit 3 if a finding at this severity or above exists (for CI)") + + sp = sub.add_parser("inspect", help="topology, cache hierarchy, THP/hugetlb and kernel config audit") + sp.add_argument("--json", action="store_true") + sp.set_defaults(func=cmd_inspect) + + sp = sub.add_parser("record", help="record raw /proc + /sys state to a JSONL file") + sp.add_argument("-d", "--duration", type=float, default=10.0) + sp.add_argument("-i", "--interval", type=float, default=1.0) + sp.add_argument("-p", "--pid", type=int, help="also record this process (stat, status, schedstat, numa_maps, smaps_rollup)") + sp.add_argument("--slow-every", type=int, default=5, help="read page-table-walking files every N samples") + sp.add_argument("-o", "--output", default="recording.jsonl.gz") + sp.set_defaults(func=cmd_record) + + sp = sub.add_parser("analyze", help="analyse a recording") + sp.add_argument("recording") + sp.add_argument("--perf-stat", help="saved `perf stat -x;` output to use as hardware counters") + outputs(sp) + sp.set_defaults(func=cmd_analyze) + + sp = sub.add_parser("profile", help="run a command under counters + recording, then analyse") + sp.add_argument("-i", "--interval", type=float, default=0.5) + sp.add_argument("-e", "--events", help="comma-separated perf events (default: a cache/TLB/branch set)") + sp.add_argument("-o", "--output", help="also save the recording") + outputs(sp) + sp.add_argument("command", nargs=argparse.REMAINDER) + sp.set_defaults(func=cmd_profile) + + sp = sub.add_parser("parse", help="analyse saved perf stat / vmstat / numastat text (v0.1 workflow)") + sp.add_argument("--perf-file", required=True) + sp.add_argument("--vmstat-file") + sp.add_argument("--numastat-file") + sp.add_argument("-o", "--output") + sp.set_defaults(func=cmd_parse) + + sp = sub.add_parser("demo", help="analyse a synthetic two-socket server scenario (no privileges needed)") + sp.add_argument("scenario", nargs="?", default="numa_misplaced") + sp.add_argument("--list", action="store_true") + sp.add_argument("--save", help="save the synthetic recording") + sp.add_argument("--json") + sp.add_argument("--md") + sp.set_defaults(func=cmd_demo) + + sp = sub.add_parser("compare", help="A/B compare reports with a permutation test") + sp.add_argument("--before", nargs="+", required=True) + sp.add_argument("--after", nargs="+", required=True) + sp.add_argument("--min-change", type=float, default=0.05) + sp.add_argument("--all", action="store_true", help="compare every numeric metric, not just the key ones") + sp.add_argument("--fail-on-regression", action="store_true") + sp.set_defaults(func=cmd_compare) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + return int(args.func(args)) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 diff --git a/src/kptk/compare.py b/src/kptk/compare.py new file mode 100644 index 0000000..a50571d --- /dev/null +++ b/src/kptk/compare.py @@ -0,0 +1,127 @@ +"""Noise-aware A/B comparison of reports. + +A single before/after pair can't tell a 3% regression from run-to-run noise. +Given several reports per side, each metric gets a two-sided permutation +test on the difference of means. It's exact for small n, distribution-free, and +needs only the standard library. With one report per side, only the relative +delta is shown and no significance is claimed. +""" + +from __future__ import annotations + +import itertools +import math +import random +import statistics +from dataclasses import dataclass +from typing import Any + +# Metrics where a higher value is better; everything else in the curated list is "lower is better". +HIGHER_IS_BETTER = {"cache.ipc", "hugepages.process_thp_coverage", "numa.hint_fault_local_frac"} +KEY_METRICS = ( + "cpu.utilization", + "cpu.psi_cpu_some", + "cpu.runq_waiting_tasks_avg", + "cpu.runq_wait_per_timeslice_ms", + "cpu.process_runq_wait_share", + "cpu.process_involuntary_switches_per_s", + "cpu.process_voluntary_switches_per_s", + "memory.faults_per_s", + "memory.major_faults_per_s", + "memory.process_minor_faults_per_s", + "memory.process_major_faults_per_s", + "memory.direct_reclaim_stalls_per_s", + "memory.psi_memory_some", + "numa.remote_alloc_frac", + "numa.process_remote_access_frac_est", + "numa.hint_fault_local_frac", + "cache.ipc", + "cache.llc_mpki", + "cache.branch_miss_ratio", + "cache.dtlb_miss_ratio", + "hugepages.process_thp_coverage", + "hugepages.thp_fallback_frac", + "hugepages.compact_stall_per_s", +) + + +def flatten(report: dict[str, Any]) -> dict[str, float]: + out: dict[str, float] = {} + for domain, metrics in report.get("metrics", {}).items(): + for k, v in metrics.items(): + if isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v): + out[f"{domain}.{k}"] = float(v) + return out + + +def permutation_p(a: list[float], b: list[float], rounds: int = 10_000, seed: int = 0) -> float: + """Two-sided p-value for difference in means. Exact enumeration when small, Monte Carlo otherwise.""" + observed = abs(statistics.fmean(b) - statistics.fmean(a)) + pooled = a + b + n = len(a) + if math.comb(len(pooled), n) <= rounds: + splits: Any = (set(c) for c in itertools.combinations(range(len(pooled)), n)) + total = extreme = 0 + for idx in splits: + xa = [pooled[i] for i in idx] + xb = [pooled[i] for i in range(len(pooled)) if i not in idx] + total += 1 + extreme += abs(statistics.fmean(xb) - statistics.fmean(xa)) >= observed - 1e-12 + return extreme / total + rng = random.Random(seed) + extreme = 0 + for _ in range(rounds): + rng.shuffle(pooled) + extreme += abs(statistics.fmean(pooled[n:]) - statistics.fmean(pooled[:n])) >= observed - 1e-12 + return (extreme + 1) / (rounds + 1) + + +@dataclass +class Change: + metric: str + before: float + after: float + rel_change: float | None + p_value: float | None + verdict: str + + +def compare(before: list[dict[str, Any]], after: list[dict[str, Any]], min_rel: float = 0.05, alpha: float = 0.05, all_metrics: bool = False) -> list[Change]: + fa = [flatten(r) for r in before] + fb = [flatten(r) for r in after] + names = sorted(set.intersection(*(set(x) for x in fa + fb))) + if not all_metrics: + names = [n for n in names if n in KEY_METRICS] + out = [] + for name in names: + xa, xb = [x[name] for x in fa], [x[name] for x in fb] + ma, mb = statistics.fmean(xa), statistics.fmean(xb) + rel = (mb - ma) / abs(ma) if ma else None + p = permutation_p(xa, xb) if len(xa) >= 2 and len(xb) >= 2 else None + significant = (p is None or p < alpha) and rel is not None and abs(rel) >= min_rel + if not significant or rel is None: + verdict = "no significant change" if p is not None else ("unchanged" if rel is not None and abs(rel) < min_rel else "changed (n=1, untested)") + else: + better = (rel > 0) == (name in HIGHER_IS_BETTER) + verdict = "improved" if better else "regressed" + if p is None: + verdict += " (n=1, untested)" + out.append(Change(name, ma, mb, rel, p, verdict)) + return out + + +def to_markdown(changes: list[Change], n_before: int, n_after: int) -> str: + lines = [ + f"# Comparison ({n_before} run(s) before, {n_after} after)", + "", + "| metric | before | after | change | p-value | verdict |", + "|---|---:|---:|---:|---:|---|", + ] + order = {"regressed": 0, "improved": 1} + for c in sorted(changes, key=lambda c: (order.get(c.verdict.split()[0], 2), c.metric)): + rel = f"{c.rel_change:+.1%}" if c.rel_change is not None else "-" + p = f"{c.p_value:.3f}" if c.p_value is not None else "-" + lines.append(f"| {c.metric} | {c.before:.4g} | {c.after:.4g} | {rel} | {p} | {c.verdict} |") + if n_before < 2 or n_after < 2: + lines += ["", "_With one run per side, changes can't be separated from noise. Record ≥ 3 runs per side for p-values._"] + return "\n".join(lines) + "\n" diff --git a/src/kptk/parsers/__init__.py b/src/kptk/parsers/__init__.py new file mode 100644 index 0000000..124c1a5 --- /dev/null +++ b/src/kptk/parsers/__init__.py @@ -0,0 +1 @@ +"""Pure text parsers for /proc, /sys and classic tool output.""" diff --git a/src/kptk/parsers/proc.py b/src/kptk/parsers/proc.py new file mode 100644 index 0000000..1b94514 --- /dev/null +++ b/src/kptk/parsers/proc.py @@ -0,0 +1,246 @@ +"""Pure parsers for /proc text formats. + +Every function takes the file's text and returns plain data, so analysis can +run on a recording captured on another machine (see :mod:`kptk.record`). +Field layouts follow Documentation/filesystems/proc.rst and +Documentation/scheduler/sched-stats.rst in the kernel tree. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +CPU_TIME_FIELDS = ("user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", "guest", "guest_nice") + + +@dataclass +class ProcStat: + """``/proc/stat``. CPU times are in USER_HZ ticks (almost always 1/100 s).""" + + cpu_total: dict[str, int] + cpus: dict[int, dict[str, int]] + ctxt: int = 0 + processes: int = 0 + procs_running: int = 0 + procs_blocked: int = 0 + intr: int = 0 + + +def parse_proc_stat(text: str) -> ProcStat: + total: dict[str, int] = {} + cpus: dict[int, dict[str, int]] = {} + scalars: dict[str, int] = {} + for line in text.splitlines(): + parts = line.split() + if not parts: + continue + key = parts[0] + if key.startswith("cpu"): + times = dict(zip(CPU_TIME_FIELDS, (int(v) for v in parts[1:]), strict=False)) + if key == "cpu": + total = times + else: + cpus[int(key[3:])] = times + elif key in ("ctxt", "processes", "procs_running", "procs_blocked"): + scalars[key] = int(parts[1]) + elif key == "intr": + scalars["intr"] = int(parts[1]) + return ProcStat(total, cpus, **scalars) + + +def busy_ticks(times: dict[str, int]) -> int: + """Non-idle ticks. guest/guest_nice are already included in user/nice, so they're excluded here.""" + return sum(v for k, v in times.items() if k not in ("idle", "iowait", "guest", "guest_nice")) + + +def total_ticks(times: dict[str, int]) -> int: + return sum(v for k, v in times.items() if k not in ("guest", "guest_nice")) + + +@dataclass +class CpuSchedStat: + """One ``cpuN`` line of /proc/schedstat (version 15+). + + ``run_delay_ns`` is the total time tasks spent *runnable but waiting* on + this CPU's run queue: direct scheduler latency, which no counter in + /proc/stat shows. + """ + + rq_cpu_time_ns: int + run_delay_ns: int + timeslices: int + + +def parse_schedstat(text: str) -> tuple[int, dict[int, CpuSchedStat]]: + version = 0 + cpus: dict[int, CpuSchedStat] = {} + for line in text.splitlines(): + parts = line.split() + if not parts: + continue + if parts[0] == "version": + version = int(parts[1]) + elif parts[0].startswith("cpu") and len(parts) >= 10: + # Fields 7-9 (1-based after the cpu label) are rq_cpu_time, run_delay, pcount. + cpus[int(parts[0][3:])] = CpuSchedStat(int(parts[7]), int(parts[8]), int(parts[9])) + return version, cpus + + +@dataclass +class TaskSchedStat: + """``/proc//schedstat``: time on CPU, time waiting on a run queue, timeslices.""" + + on_cpu_ns: int + wait_ns: int + timeslices: int + + +def parse_task_schedstat(text: str) -> TaskSchedStat: + a, b, c = (int(x) for x in text.split()[:3]) + return TaskSchedStat(a, b, c) + + +@dataclass +class PidStat: + pid: int + comm: str + state: str + minflt: int + majflt: int + utime: int + stime: int + num_threads: int + processor: int + rss_pages: int + + +def parse_pid_stat(text: str) -> PidStat: + """``/proc//stat``. ``comm`` may contain spaces and parentheses, so split at the *last* ')'.""" + lpar, rpar = text.index("("), text.rindex(")") + pid = int(text[:lpar].strip()) + comm = text[lpar + 1 : rpar] + f = text[rpar + 2 :].split() + # f[0] is field 3 (state); field n is f[n - 3]. + return PidStat( + pid=pid, + comm=comm, + state=f[0], + minflt=int(f[7]), + majflt=int(f[9]), + utime=int(f[11]), + stime=int(f[12]), + num_threads=int(f[17]), + rss_pages=int(f[21]), + processor=int(f[36]), + ) + + +def parse_key_value(text: str, sep: str | None = None) -> dict[str, str]: + out: dict[str, str] = {} + for line in text.splitlines(): + parts = line.split(None, 1) if sep is None else line.split(sep, 1) + if len(parts) == 2: + out[parts[0].strip().rstrip(":")] = parts[1].strip() + return out + + +def parse_pid_status(text: str) -> dict[str, str | int]: + """``/proc//status``. Numeric ``kB`` fields become ints in kB; other values are kept as strings.""" + out: dict[str, str | int] = {} + for key, value in parse_key_value(text, ":").items(): + token = value.split()[0] if value else "" + out[key] = int(token) if token.isdigit() else value + return out + + +def parse_vmstat(text: str) -> dict[str, int]: + """``/proc/vmstat``: ``name value`` pairs.""" + out: dict[str, int] = {} + for line in text.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].lstrip("-").isdigit(): + out[parts[0]] = int(parts[1]) + return out + + +def parse_meminfo(text: str) -> dict[str, int]: + """``/proc/meminfo`` (and per-node meminfo). Values in kB, or plain counts for HugePages_*.""" + out: dict[str, int] = {} + for line in text.splitlines(): + if ":" not in line: + continue + key, rest = line.split(":", 1) + key = key.split()[-1] # per-node files are prefixed "Node 0 " + if not key[:1].isalpha(): # smaps_rollup's address-range header line + continue + token = rest.split()[0] if rest.split() else "" + if token.isdigit(): + out[key] = int(token) + return out + + +@dataclass +class Pressure: + """PSI line: share of wall time in which *some* (or *all*, for full) tasks were stalled.""" + + avg10: float + avg60: float + avg300: float + total_us: int + + +def parse_pressure(text: str) -> dict[str, Pressure]: + out: dict[str, Pressure] = {} + for line in text.splitlines(): + parts = line.split() + if not parts: + continue + kv = dict(p.split("=", 1) for p in parts[1:]) + out[parts[0]] = Pressure(float(kv["avg10"]), float(kv["avg60"]), float(kv["avg300"]), int(kv["total"])) + return out + + +@dataclass +class NumaMaps: + """Aggregated ``/proc//numa_maps``: where a process's pages physically live.""" + + pages_by_node: dict[int, int] = field(default_factory=dict) + huge_pages_by_node: dict[int, int] = field(default_factory=dict) + anon_pages: int = 0 + policies: dict[str, int] = field(default_factory=dict) + + @property + def total_pages(self) -> int: + return sum(self.pages_by_node.values()) + + +def parse_numa_maps(text: str) -> NumaMaps: + out = NumaMaps() + for line in text.splitlines(): + parts = line.split() + if len(parts) < 2: + continue + policy = parts[1].split(":")[0] + out.policies[policy] = out.policies.get(policy, 0) + 1 + page_kb = 4 + nodes: dict[int, int] = {} + for token in parts[2:]: + if token.startswith("kernelpagesize_kB="): + page_kb = int(token.split("=")[1]) + elif token.startswith("anon="): + out.anon_pages += int(token.split("=")[1]) + elif token.startswith("N") and "=" in token and token[1:].split("=")[0].isdigit(): + node_s, count_s = token[1:].split("=") + nodes[int(node_s)] = int(count_s) + # Normalise huge pages to 4 KiB-equivalents so nodes are comparable by bytes. + scale = page_kb // 4 + for n, count in nodes.items(): + out.pages_by_node[n] = out.pages_by_node.get(n, 0) + count * scale + if page_kb > 4: + out.huge_pages_by_node[n] = out.huge_pages_by_node.get(n, 0) + count + return out + + +def parse_smaps_rollup(text: str) -> dict[str, int]: + """``/proc//smaps_rollup``: kB totals (Rss, Anonymous, AnonHugePages, Swap, ...).""" + return parse_meminfo(text) diff --git a/src/kptk/parsers/sysfs.py b/src/kptk/parsers/sysfs.py new file mode 100644 index 0000000..cb6a83c --- /dev/null +++ b/src/kptk/parsers/sysfs.py @@ -0,0 +1,37 @@ +"""Parsers for /sys formats: CPU lists, THP modes, cache sizes, NUMA distances.""" + +from __future__ import annotations + + +def parse_cpulist(text: str) -> list[int]: + """``0-3,8-11`` -> [0, 1, 2, 3, 8, 9, 10, 11].""" + out: list[int] = [] + for part in text.strip().split(","): + if not part: + continue + if "-" in part: + lo, hi = part.split("-") + out.extend(range(int(lo), int(hi) + 1)) + else: + out.append(int(part)) + return out + + +def parse_bracket_choice(text: str) -> str | None: + """THP-style mode files: ``always [madvise] never`` -> ``madvise``.""" + for token in text.split(): + if token.startswith("[") and token.endswith("]"): + return token[1:-1] + return None + + +def parse_size(text: str) -> int: + """Cache size strings: ``48K`` / ``2048K`` / ``32M`` -> bytes.""" + t = text.strip().upper() + mult = {"K": 1 << 10, "M": 1 << 20, "G": 1 << 30}.get(t[-1:], 1) + return int(t.rstrip("KMG")) * mult + + +def parse_distance(text: str) -> list[int]: + """``/sys/devices/system/node/nodeN/distance``: SLIT distances (10 = local).""" + return [int(x) for x in text.split()] diff --git a/src/kptk/parsers/tools.py b/src/kptk/parsers/tools.py new file mode 100644 index 0000000..618b506 --- /dev/null +++ b/src/kptk/parsers/tools.py @@ -0,0 +1,102 @@ +"""Parsers for the text output of ``perf stat``, ``vmstat`` and ``numastat``. + +These support the offline workflow from v0.1: analyse output someone pasted +into a ticket. Live collection reads /proc and perf_event_open directly instead. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_NUMBER = re.compile(r"^([0-9][0-9,]*(?:\.[0-9]+)?)([KMG]?)$") + + +def _number(value: str) -> float: + match = _NUMBER.match(value.strip()) + if not match: + raise ValueError(f"invalid numeric value: {value!r}") + amount = float(match.group(1).replace(",", "")) + return amount * {"": 1, "K": 1e3, "M": 1e6, "G": 1e9}[match.group(2)] + + +@dataclass +class PerfStatEvent: + value: float + running_pct: float = 100.0 # share of enabled time the counter was scheduled on the PMU + + +def parse_perf_stat(text: str) -> dict[str, PerfStatEvent]: + """Parse ``perf stat`` output, CSV (``-x ;`` or ``-x ,``) or human-readable. + + CSV columns: value; unit; event; run-time; running-pct; ... ``perf stat`` + already scales multiplexed counts. The running percentage is kept so callers + know how much of the value is extrapolation. ```` and + ```` rows are skipped. + """ + out: dict[str, PerfStatEvent] = {} + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "= 2 and not re.match(r"^\s*[\d,]+\s+\S", line) else None) + if sep: + f = [x.strip() for x in line.split(sep)] + if len(f) >= 3 and f[0] and f[2]: + try: + pct = float(f[4]) if len(f) > 4 and f[4] else 100.0 + out[f[2]] = PerfStatEvent(_number(f[0]), pct) + except ValueError: + pass + continue + f = line.split() + if len(f) < 2: + continue + try: + value = _number(f[0]) + except ValueError: + continue + idx = 2 if len(f) > 2 and f[1] in {"msec", "seconds", "ns"} else 1 + pct = 100.0 + m = re.search(r"\((\d+(?:\.\d+)?)%\)", line) + if m: + pct = float(m.group(1)) + out[f[idx]] = PerfStatEvent(value, pct) + return out + + +def parse_vmstat_output(text: str) -> dict[str, float]: + """``vmstat N`` output; returns the last sample row.""" + headers: list[str] | None = None + latest: dict[str, float] = {} + for raw in text.splitlines(): + f = raw.split() + if not f: + continue + if f[0] == "procs": + continue + if f[0] == "r" and "free" in f: + headers = f + continue + if headers and len(f) >= len(headers) and all(x.lstrip("-").isdigit() for x in f[: len(headers)]): + latest = {k: float(v) for k, v in zip(headers, f, strict=False)} + return latest + + +def parse_numastat_output(text: str) -> dict[str, dict[str, float]]: + """``numastat`` output as metric -> node -> value (plus ``total``).""" + result: dict[str, dict[str, float]] = {} + nodes: list[str] = [] + for raw in text.splitlines(): + f = raw.split() + if f and all(re.fullmatch(r"node\d+", x.lower()) for x in f): + nodes = [x.lower() for x in f] + continue + if len(f) < 2 or not all(re.fullmatch(r"[0-9.,]+", v) for v in f[1:]): + continue + values = [float(v.replace(",", "")) for v in f[1:]] + if nodes and len(values) == len(nodes): + result[f[0]] = {**dict(zip(nodes, values, strict=True)), "total": sum(values)} + elif len(values) >= 2: + result[f[0]] = {**{f"node{i}": v for i, v in enumerate(values[:-1])}, "total": values[-1]} + return result diff --git a/src/kptk/perf_event.py b/src/kptk/perf_event.py new file mode 100644 index 0000000..aab2006 --- /dev/null +++ b/src/kptk/perf_event.py @@ -0,0 +1,266 @@ +"""Hardware and software counters through the ``perf_event_open(2)`` syscall, via ctypes. + +No ``perf`` binary or kernel headers are needed: the attr struct, constants and +ioctls are defined here from ``include/uapi/linux/perf_event.h``. + +Multiplexing: a CPU has a handful of programmable PMU counters (often 4-8 per +hyperthread). When more events are requested, the kernel time-slices them, and +each counter reports ``time_enabled`` and ``time_running``. The estimated full count is +``value * enabled / running``, an extrapolation that assumes the workload is +uniform over time. Grouped events are always scheduled together, so *ratios within a +group* (IPC, miss rate) stay exact even under multiplexing. That's why +:class:`CounterGroup` keeps leader-relative ratios in one group when it can. + +Access is governed by ``/proc/sys/kernel/perf_event_paranoid`` (and +CAP_PERFMON). On Ubuntu the default of 4 blocks unprivileged use entirely, so every +entry point here degrades to a clear :class:`PerfUnavailable` error instead of crashing. +""" + +from __future__ import annotations + +import ctypes +import ctypes.util +import errno +import os +import platform +import struct +from dataclasses import dataclass + +# perf_type_id +TYPE_HARDWARE, TYPE_SOFTWARE, TYPE_HW_CACHE = 0, 1, 3 +# perf_hw_id +HW = { + "cycles": 0, + "instructions": 1, + "cache-references": 2, + "cache-misses": 3, + "branches": 4, + "branch-misses": 5, + "stalled-cycles-frontend": 7, + "stalled-cycles-backend": 8, + "ref-cycles": 9, +} +# perf_sw_ids +SW = { + "cpu-clock": 0, + "task-clock": 1, + "page-faults": 2, + "context-switches": 3, + "cpu-migrations": 4, + "minor-faults": 5, + "major-faults": 6, +} +# HW cache: id | (op << 8) | (result << 16) +CACHE_ID = {"L1-dcache": 0, "L1-icache": 1, "LLC": 2, "dTLB": 3, "iTLB": 4, "branch": 5, "node": 6} +# perf's naming: "-loads" counts accesses, "-load-misses" counts misses. +CACHE_SUFFIX = { + "loads": (0, 0), + "load-misses": (0, 1), + "stores": (1, 0), + "store-misses": (1, 1), + "prefetches": (2, 0), + "prefetch-misses": (2, 1), +} + +FORMAT_TOTAL_TIME_ENABLED, FORMAT_TOTAL_TIME_RUNNING, FORMAT_ID, FORMAT_GROUP = 1, 2, 4, 8 +FLAG_DISABLED, FLAG_INHERIT, FLAG_EXCLUDE_KERNEL, FLAG_EXCLUDE_HV, FLAG_ENABLE_ON_EXEC = 1 << 0, 1 << 1, 1 << 5, 1 << 6, 1 << 12 +IOC_ENABLE, IOC_DISABLE, IOC_RESET, IOC_FLAG_GROUP = 0x2400, 0x2401, 0x2403, 1 + +SYSCALL_NR = {"x86_64": 298, "aarch64": 241, "riscv64": 241, "ppc64le": 319, "s390x": 331, "i686": 336} +PERF_ATTR_SIZE_VER5 = 112 + + +class PerfEventAttr(ctypes.Structure): + """``struct perf_event_attr`` up to PERF_ATTR_SIZE_VER5 (112 bytes, Linux 4.1+). + + Newer kernels accept older sizes, so declaring the minimum we need keeps + this working everywhere. + """ + + _fields_ = [ + ("type", ctypes.c_uint32), + ("size", ctypes.c_uint32), + ("config", ctypes.c_uint64), + ("sample_period", ctypes.c_uint64), + ("sample_type", ctypes.c_uint64), + ("read_format", ctypes.c_uint64), + ("flags", ctypes.c_uint64), + ("wakeup_events", ctypes.c_uint32), + ("bp_type", ctypes.c_uint32), + ("config1", ctypes.c_uint64), + ("config2", ctypes.c_uint64), + ("branch_sample_type", ctypes.c_uint64), + ("sample_regs_user", ctypes.c_uint64), + ("sample_stack_user", ctypes.c_uint32), + ("clockid", ctypes.c_int32), + ("sample_regs_intr", ctypes.c_uint64), + ("aux_watermark", ctypes.c_uint32), + ("sample_max_stack", ctypes.c_uint16), + ("reserved_2", ctypes.c_uint16), + ] + + +class PerfUnavailable(RuntimeError): + pass + + +def event_config(name: str) -> tuple[int, int]: + """Map a perf-style event name to ``(type, config)``.""" + if name in HW: + return TYPE_HARDWARE, HW[name] + if name in SW: + return TYPE_SOFTWARE, SW[name] + for cache, cid in CACHE_ID.items(): + if name.startswith(cache + "-") and name[len(cache) + 1 :] in CACHE_SUFFIX: + op, result = CACHE_SUFFIX[name[len(cache) + 1 :]] + return TYPE_HW_CACHE, cid | (op << 8) | (result << 16) + raise ValueError(f"unknown event {name!r}") + + +def make_attr( + name: str, *, disabled: bool = True, inherit: bool = False, enable_on_exec: bool = False, user_only: bool = False, group: bool = False +) -> PerfEventAttr: + etype, config = event_config(name) + attr = PerfEventAttr() + attr.type, attr.size, attr.config = etype, ctypes.sizeof(PerfEventAttr), config + attr.read_format = FORMAT_TOTAL_TIME_ENABLED | FORMAT_TOTAL_TIME_RUNNING | (FORMAT_GROUP | FORMAT_ID if group else 0) + flags = FLAG_EXCLUDE_HV + flags |= FLAG_DISABLED if disabled else 0 + flags |= FLAG_INHERIT if inherit else 0 + flags |= FLAG_ENABLE_ON_EXEC if enable_on_exec else 0 + flags |= FLAG_EXCLUDE_KERNEL if user_only else 0 + attr.flags = flags + return attr + + +def scale(value: int, enabled: int, running: int) -> tuple[float, float]: + """Multiplexing-corrected count and the fraction of time actually counted.""" + if running == 0: + return 0.0, 0.0 + return value * enabled / running, running / enabled if enabled else 0.0 + + +_libc: ctypes.CDLL | None = None + + +def _syscall(attr: PerfEventAttr, pid: int, cpu: int, group_fd: int, flags: int) -> int: + global _libc + nr = SYSCALL_NR.get(platform.machine()) + if nr is None: + raise PerfUnavailable(f"perf_event_open syscall number unknown for {platform.machine()}") + if _libc is None: + _libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) + fd = int(_libc.syscall(nr, ctypes.byref(attr), pid, cpu, group_fd, ctypes.c_ulong(flags))) + if fd < 0: + err = ctypes.get_errno() + hint = "" + if err in (errno.EACCES, errno.EPERM): + hint = f" (perf_event_paranoid={_paranoid()}; need <= 2 for own processes, or CAP_PERFMON)" + elif err == errno.ENOENT: + hint = " (event not supported by this CPU/PMU, common in VMs)" + raise PerfUnavailable(f"perf_event_open failed: {os.strerror(err)}{hint}") + return fd + + +def _ioctl(fd: int, request: int, arg: int = 0) -> None: + assert _libc is not None + if _libc.ioctl(fd, request, arg) < 0: + raise OSError(ctypes.get_errno(), "perf ioctl failed") + + +def _paranoid() -> str: + try: + with open("/proc/sys/kernel/perf_event_paranoid") as fh: + return fh.read().strip() + except OSError: + return "?" + + +@dataclass +class Reading: + name: str + raw: int + enabled_ns: int + running_ns: int + + @property + def value(self) -> float: + return scale(self.raw, self.enabled_ns, self.running_ns)[0] + + @property + def running_frac(self) -> float: + return scale(self.raw, self.enabled_ns, self.running_ns)[1] + + +class Counters: + """Independent (ungrouped) counters on a task, optionally inherited by its children. + + ``inherit`` is needed to count all threads and children of a command, and + the kernel doesn't support PERF_FORMAT_GROUP reads on inherited events, so + these are separate fds. Each is multiplex-scaled independently. + """ + + def __init__(self, events: list[str], pid: int = 0, cpu: int = -1, inherit: bool = True, enable_on_exec: bool = False) -> None: + self.events = events + self.fds: list[tuple[str, int]] = [] + self.skipped: dict[str, str] = {} + for name in events: + attr = make_attr(name, disabled=True, inherit=inherit, enable_on_exec=enable_on_exec) + try: + self.fds.append((name, _syscall(attr, pid, cpu, -1, 0))) + except PerfUnavailable as exc: + if not self.fds and "paranoid" in str(exc): + raise # no permission at all: fail fast with the hint + self.skipped[name] = str(exc) + + def enable(self) -> None: + for _, fd in self.fds: + _ioctl(fd, IOC_ENABLE) + + def disable(self) -> None: + for _, fd in self.fds: + _ioctl(fd, IOC_DISABLE) + + def read(self) -> list[Reading]: + out = [] + for name, fd in self.fds: + value, enabled, running = struct.unpack("QQQ", os.read(fd, 24)) + out.append(Reading(name, value, enabled, running)) + return out + + def close(self) -> None: + for _, fd in self.fds: + os.close(fd) + self.fds = [] + + def __enter__(self) -> Counters: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +def readings_to_dict(readings: list[Reading], skipped: dict[str, str] | None = None) -> dict[str, object]: + return { + "events": {r.name: {"value": r.value, "raw": r.raw, "running_frac": round(r.running_frac, 4)} for r in readings}, + "skipped": skipped or {}, + } + + +DEFAULT_EVENTS = [ + "cycles", + "instructions", + "cache-references", + "cache-misses", + "branches", + "branch-misses", + "L1-dcache-loads", + "L1-dcache-load-misses", + "dTLB-loads", + "dTLB-load-misses", + "task-clock", + "context-switches", + "cpu-migrations", + "minor-faults", + "major-faults", +] diff --git a/src/kptk/profile.py b/src/kptk/profile.py new file mode 100644 index 0000000..d5ca7f0 --- /dev/null +++ b/src/kptk/profile.py @@ -0,0 +1,64 @@ +"""Profile a command: hardware counters plus /proc recording for its whole lifetime. + +The counters must cover the command from its first instruction, without counting the +profiler. That's the classic ``perf stat`` sequence: + +1. fork; the child blocks reading a pipe *before* exec, +2. the parent opens counters on the child pid with ``disabled=1, enable_on_exec=1, inherit=1``, +3. the parent writes to the pipe; the child execs, and the kernel enables the counters at exec. + +``subprocess.Popen`` can't do this: it blocks until the child has exec'd +(it waits on an internal status pipe), but here the child is deliberately +held *before* exec until the parent has attached the counters, so Popen +deadlocks. Plain ``os.fork`` / ``os.execvp`` avoids that. + +Exit detection uses ``waitid(WNOWAIT)`` so the child stays a zombie until the +recorder takes its closing sample: /proc//stat of a zombie still holds the +final fault and CPU-time totals. +""" + +from __future__ import annotations + +import os + +from .perf_event import DEFAULT_EVENTS, Counters, PerfUnavailable, readings_to_dict +from .record import Recorder, Recording + + +def _exited(pid: int) -> bool: + try: + return os.waitid(os.P_PID, pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) is not None + except ChildProcessError: + return True + + +def profile(cmd: list[str], interval_s: float = 0.5, events: list[str] | None = None, slow_every: int = 4) -> tuple[Recording, int]: + rfd, wfd = os.pipe() + pid = os.fork() + if pid == 0: # child: wait for the go signal, then become the command + try: + os.close(wfd) + os.read(rfd, 1) + os.close(rfd) + os.execvp(cmd[0], cmd) + except BaseException: + os._exit(127) + os.close(rfd) + counters: Counters | None = None + error = None + try: + counters = Counters(events or DEFAULT_EVENTS, pid=pid, inherit=True, enable_on_exec=True) + except PerfUnavailable as exc: + error = str(exc) + os.write(wfd, b"g") + os.close(wfd) + + rec = Recorder(interval_s=interval_s, pid=pid, slow_every=slow_every).run(until=lambda: _exited(pid), command=cmd) + _, status = os.waitpid(pid, 0) + returncode = os.waitstatus_to_exitcode(status) + if counters is not None: + rec.counters = readings_to_dict(counters.read(), counters.skipped) + counters.close() + else: + rec.counters = {"events": None, "error": error} + return rec, returncode diff --git a/src/kptk/record.py b/src/kptk/record.py new file mode 100644 index 0000000..b486ce0 --- /dev/null +++ b/src/kptk/record.py @@ -0,0 +1,278 @@ +"""Capture raw /proc and /sys text over time; analyse later, anywhere. + +Design: the recorder never parses. It copies file contents verbatim into a +JSONL recording, which keeps per-sample overhead to a few hundred +microseconds of read() syscalls. It also means a recording taken on a +production host can be analysed on a laptop, re-analysed after a parser bug +fix, or attached to a ticket. Parsing and analysis are pure functions over +the recording (see :mod:`kptk.analysis`). + +Expensive files (``numa_maps`` and ``smaps_rollup`` walk the target's page +tables under its mmap lock) are read only every ``slow_every`` samples, so +the recorder doesn't perturb the process it's watching. +""" + +from __future__ import annotations + +import fnmatch +import glob +import gzip +import json +import os +import platform +import threading +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from pathlib import Path +from typing import IO, Any + +from . import __version__ + +STATIC_PATTERNS = ( + "/proc/version", + "/proc/sys/kernel/numa_balancing", + "/proc/sys/kernel/perf_event_paranoid", + "/proc/sys/vm/swappiness", + "/proc/sys/vm/zone_reclaim_mode", + "/proc/sys/vm/overcommit_memory", + "/proc/sys/vm/min_free_kbytes", + "/sys/kernel/mm/transparent_hugepage/enabled", + "/sys/kernel/mm/transparent_hugepage/defrag", + "/sys/kernel/mm/transparent_hugepage/khugepaged/defrag", + "/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none", + "/sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan", + "/sys/kernel/mm/transparent_hugepage/khugepaged/scan_sleep_millisecs", + "/sys/devices/system/cpu/online", + "/sys/devices/system/node/online", + "/sys/devices/system/cpu/cpu[0-9]*/topology/physical_package_id", + "/sys/devices/system/cpu/cpu[0-9]*/topology/core_id", + "/sys/devices/system/cpu/cpu[0-9]*/topology/thread_siblings_list", + "/sys/devices/system/cpu/cpu[0-9]*/cache/index[0-9]*/level", + "/sys/devices/system/cpu/cpu[0-9]*/cache/index[0-9]*/type", + "/sys/devices/system/cpu/cpu[0-9]*/cache/index[0-9]*/size", + "/sys/devices/system/cpu/cpu[0-9]*/cache/index[0-9]*/shared_cpu_list", + "/sys/devices/system/cpu/cpu[0-9]*/cache/index[0-9]*/coherency_line_size", + "/sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_governor", + "/sys/devices/system/node/node[0-9]*/cpulist", + "/sys/devices/system/node/node[0-9]*/distance", + "/sys/kernel/mm/hugepages/hugepages-*/nr_hugepages", + "/sys/kernel/mm/hugepages/hugepages-*/free_hugepages", + "/sys/kernel/mm/hugepages/hugepages-*/resv_hugepages", + "/sys/kernel/mm/hugepages/hugepages-*/surplus_hugepages", +) + +DYNAMIC_PATTERNS = ( + "/proc/stat", + "/proc/schedstat", + "/proc/vmstat", + "/proc/meminfo", + "/proc/loadavg", + "/proc/pressure/cpu", + "/proc/pressure/memory", + "/proc/pressure/io", + "/sys/devices/system/node/node[0-9]*/numastat", + "/sys/devices/system/node/node[0-9]*/meminfo", +) + +PID_FAST = ("stat", "status", "schedstat") +PID_SLOW = ("numa_maps", "smaps_rollup") + + +class Source: + """Where file contents come from: the live system, or a dict (tests, replays).""" + + def read(self, path: str) -> str | None: + raise NotImplementedError + + def glob(self, pattern: str) -> list[str]: + raise NotImplementedError + + +class LiveSource(Source): + def __init__(self, root: str = "/") -> None: + self.root = root.rstrip("/") + + def read(self, path: str) -> str | None: + try: + with open(self.root + path, encoding="utf-8", errors="replace") as fh: + return fh.read() + except OSError: # vanished pid, permission denied, attribute unsupported on this kernel + return None + + def glob(self, pattern: str) -> list[str]: + n = len(self.root) + return sorted(p[n:] for p in glob.glob(self.root + pattern)) + + +class DictSource(Source): + def __init__(self, files: dict[str, str]) -> None: + self.files = files + + def read(self, path: str) -> str | None: + return self.files.get(path) + + def glob(self, pattern: str) -> list[str]: + return sorted(p for p in self.files if fnmatch.fnmatchcase(p, pattern)) + + +def expand(source: Source, patterns: Iterable[str]) -> list[str]: + out: list[str] = [] + for pat in patterns: + out.extend(source.glob(pat) if any(c in pat for c in "*?[") else [pat]) + return out + + +@dataclass +class Sample: + ts: float + files: dict[str, str] + cost_s: float = 0.0 + + +@dataclass +class Recording: + meta: dict[str, Any] + static: dict[str, str] + samples: list[Sample] = field(default_factory=list) + counters: dict[str, Any] | None = None + + @property + def pid(self) -> int | None: + pid = self.meta.get("pid") + return int(pid) if pid else None + + @property + def duration_s(self) -> float: + return self.samples[-1].ts - self.samples[0].ts if len(self.samples) > 1 else 0.0 + + def latest(self, path: str) -> str | None: + """Most recent sample that contains ``path`` (slow files aren't in every sample).""" + for s in reversed(self.samples): + if path in s.files: + return s.files[path] + return None + + def save(self, path: str | Path) -> None: + with _open(path, "w") as fh: + fh.write(json.dumps({"type": "meta", **self.meta, "static": self.static}) + "\n") + for s in self.samples: + fh.write(json.dumps({"type": "sample", "ts": s.ts, "cost_s": s.cost_s, "files": s.files}, separators=(",", ":")) + "\n") + if self.counters is not None: + fh.write(json.dumps({"type": "counters", **self.counters}) + "\n") + + @classmethod + def load(cls, path: str | Path) -> Recording: + meta: dict[str, Any] = {} + static: dict[str, str] = {} + samples: list[Sample] = [] + counters = None + with _open(path, "r") as fh: + for line in fh: + if not line.strip(): + continue + d = json.loads(line) + kind = d.pop("type") + if kind == "meta": + static = d.pop("static", {}) + meta = d + elif kind == "sample": + samples.append(Sample(d["ts"], d["files"], d.get("cost_s", 0.0))) + elif kind == "counters": + counters = d + return cls(meta, static, samples, counters) + + +def _open(path: str | Path, mode: str) -> IO[str]: + p = Path(path) + if p.suffix == ".gz": + return gzip.open(p, mode + "t", encoding="utf-8") # type: ignore[return-value] + return p.open(mode, encoding="utf-8") + + +class Recorder: + """Deadline-scheduled sampler (no drift; overruns are skipped and counted, not burst).""" + + def __init__( + self, + source: Source | None = None, + interval_s: float = 1.0, + pid: int | None = None, + slow_every: int = 5, + clock: Callable[[], float] = time.time, + ) -> None: + self.source = source or LiveSource() + self.interval_s = interval_s + self.pid = pid + self.slow_every = max(1, slow_every) + self.clock = clock + self.stop_event = threading.Event() + self.overruns = 0 + self._dynamic = expand(self.source, DYNAMIC_PATTERNS) + + def capture_static(self) -> dict[str, str]: + out = {} + for p in expand(self.source, STATIC_PATTERNS): + text = self.source.read(p) + if text is not None: + out[p] = text + return out + + def sample(self, index: int) -> Sample: + t0 = time.perf_counter() + ts = self.clock() + files: dict[str, str] = {} + paths = list(self._dynamic) + if self.pid: + paths += [f"/proc/{self.pid}/{n}" for n in PID_FAST] + if index % self.slow_every == 0: + paths += [f"/proc/{self.pid}/{n}" for n in PID_SLOW] + for p in paths: + text = self.source.read(p) + if text is not None: + files[p] = text + return Sample(ts, files, time.perf_counter() - t0) + + def meta(self, command: list[str] | None = None) -> dict[str, Any]: + return { + "tool": f"kptk {__version__}", + "host": platform.node(), + "kernel": platform.release(), + "arch": platform.machine(), + "started": self.clock(), + "interval_s": self.interval_s, + "pid": self.pid, + "command": command, + "user_hz": os.sysconf("SC_CLK_TCK"), + "page_size": os.sysconf("SC_PAGE_SIZE"), + } + + def run( + self, + duration_s: float | None = None, + max_samples: int | None = None, + until: Callable[[], bool] | None = None, + command: list[str] | None = None, + ) -> Recording: + rec = Recording(self.meta(command), self.capture_static()) + start = deadline = time.monotonic() + i = 0 + while not self.stop_event.is_set(): + rec.samples.append(self.sample(i)) + i += 1 + if max_samples is not None and i >= max_samples: + break + if until is not None and until(): + rec.samples.append(self.sample(0)) # closing sample, including slow files if still readable + break + if duration_s is not None and time.monotonic() - start >= duration_s: + break + deadline += self.interval_s + now = time.monotonic() + if now > deadline: + missed = int((now - deadline) // self.interval_s) + 1 + self.overruns += missed + deadline += missed * self.interval_s + self.stop_event.wait(max(0.0, deadline - time.monotonic())) + rec.meta["overruns"] = self.overruns + return rec diff --git a/src/kptk/report.py b/src/kptk/report.py new file mode 100644 index 0000000..804e611 --- /dev/null +++ b/src/kptk/report.py @@ -0,0 +1,100 @@ +"""Markdown rendering of an analysis report.""" + +from __future__ import annotations + +from typing import Any + +BLOCKS = "▁▂▃▄▅▆▇█" +ICON = {"critical": "🔴", "warning": "🟠", "info": "🔵"} + + +def sparkline(values: list[float | None], width: int = 60) -> str: + vals = [v for v in values if v is not None] + if not vals: + return "" + step = max(1, len(values) // width) + binned: list[float | None] = [] + for i in range(0, len(values), step): + chunk = [v for v in values[i : i + step] if v is not None] + binned.append(max(chunk) if chunk else None) + lo, hi = min(vals), max(vals) + span = (hi - lo) or 1.0 + return "".join(" " if v is None else BLOCKS[min(7, int((v - lo) / span * 8))] for v in binned) + + +def _fmt(v: Any) -> str: + if v is None: + return "-" + if isinstance(v, bool): + return str(v) + if isinstance(v, float): + if abs(v) >= 1000: + return f"{v:,.0f}" + return f"{v:.3g}" + if isinstance(v, dict): + return ", ".join(f"{k}: {_fmt(x)}" for k, x in v.items() if x is not None) or "-" + return str(v) + + +def render(report: dict[str, Any]) -> str: + meta, topo, dq = report["meta"], report["topology"], report["data_quality"] + caches = ", ".join(f"L{c['level']}{c['type'][0] if c['type'] != 'Unified' else ''} {c['size_kib']} KiB" for c in topo["caches"]) + out = [ + "# Kernel performance report", + "", + f"**Host** {meta.get('host')} · kernel {meta.get('kernel')} · {topo['cpus']} CPUs ({topo['physical_cores']} cores, " + f"{topo['packages']} socket(s), SMT {'on' if topo['smt'] else 'off'}) · {topo['numa_nodes']} NUMA node(s) · {caches}", + "", + f"**Window** {dq['duration_s']} s, {dq['samples']} samples, recorder cost {dq['mean_sample_cost_ms']} ms/sample" + + (f" · **target** pid {meta['pid']} ({report['metrics']['cpu'].get('process_comm', '?')})" if meta.get("pid") else "") + + (f" · `{' '.join(meta['command'])}`" if meta.get("command") else ""), + "", + ] + findings = report["findings"] + out += ["## Findings", ""] + if not any(f["severity"] != "info" for f in findings): + out += ["No warnings: nothing crossed a threshold in this window.", ""] + for f in findings: + out.append(f"### {ICON[f['severity']]} {f['title']}") + out.append("") + if f["evidence"]: + out.append("Evidence: " + ", ".join(f"`{k}={{{_fmt(v)}}}`" if isinstance(v, dict) else f"`{k}={_fmt(v)}`" for k, v in f["evidence"].items())) + out.append("") + if f.get("why"): + out.append(f"_Why it matters:_ {f['why']}") + out.append("") + out.append(f"**Action:** {f['recommendation']}") + out.append("") + + out += ["## USE summary", "", "| resource | utilisation | saturation | errors |", "|---|---:|---|---|"] + for row in report["use"]: + out.append(f"| {row['resource']} | {_fmt(row['utilization'])} | {_fmt(row['saturation'])} | {_fmt(row['errors'])} |") + out.append("") + + ts = report.get("timeseries", {}) + if ts.get("t"): + out += ["## Over time", "", "```"] + for key, label in ( + ("cpu_util", "cpu util "), + ("runq_waiting", "runq waiting "), + ("faults_per_s", "faults/s "), + ("major_faults_per_s", "major flt/s "), + ("psi_mem_some", "mem pressure "), + ): + vals = ts.get(key, []) + if any(v is not None for v in vals): + present = [v for v in vals if v is not None] + out.append(f"{label} {sparkline(vals)} max {_fmt(max(present))}") + out += ["```", ""] + + out += ["## Metrics", ""] + for domain, metrics in report["metrics"].items(): + rows = [(k, v) for k, v in metrics.items() if not isinstance(v, (dict, list)) and v is not None] + if not rows: + continue + out += [f"
{domain}", "", "| metric | value |", "|---|---:|"] + out += [f"| {k} | {_fmt(v)} |" for k, v in rows] + out += ["", "
", ""] + if dq["missing"]: + out += ["## Data quality", ""] + [f"- {x}" for x in dq["missing"]] + [""] + return "\n".join(out) diff --git a/src/kptk/synth.py b/src/kptk/synth.py new file mode 100644 index 0000000..59d9362 --- /dev/null +++ b/src/kptk/synth.py @@ -0,0 +1,387 @@ +"""Synthetic recordings of a two-socket server, for tests and ``kptk demo``. + +Generates internally consistent /proc and /sys *text* (the same formats the +recorder captures), so the full parse -> analyse path is exercised. This lets +every finding be tested deterministically and demonstrated on a single-node +laptop. Real recordings in ``samples/`` show what live data looks like. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace + +from .record import Recording, Sample + +USER_HZ = 100 + + +@dataclass +class Spec: + sockets: int = 2 + cores_per_socket: int = 8 + smt: int = 2 + duration_s: int = 10 + interval_s: float = 1.0 + mem_per_node_gib: int = 64 + # system + cpu_util: float = 0.40 + steal: float = 0.0 + runq_waiting_tasks: float = 0.05 + psi_cpu_some: float = 0.01 + psi_mem_some: float = 0.0 + psi_mem_full: float = 0.0 + faults_per_s: float = 20_000 + major_faults_per_s: float = 0.0 + allocstall_per_s: float = 0.0 + swap_per_s: float = 0.0 + node_free_frac: tuple[float, ...] = (0.5, 0.5) + allocs_per_s: float = 50_000 + remote_alloc_frac: float = 0.02 + numa_balancing: str = "1" + hint_faults_per_s: float = 0.0 + hint_local_frac: float = 0.95 + migrated_per_s: float = 0.0 + zone_reclaim_mode: str = "0" + # huge pages + thp_enabled: str = "madvise" + thp_defrag: str = "madvise" + thp_alloc_per_s: float = 10 + thp_fallback_per_s: float = 0 + compact_stall_per_s: float = 0 + hugetlb_2m_nr: int = 0 + hugetlb_2m_free: int = 0 + # target process + pid: int | None = 4242 + comm: str = "kvstore" + threads: int = 32 + proc_cpus: float = 4.0 + proc_runq_wait_share: float = 0.02 + proc_vol_cs_per_s: float = 2_000 + proc_invol_cs_per_s: float = 100 + proc_minflt_per_s: float = 2_000 + proc_majflt_per_s: float = 0.0 + proc_anon_gib: float = 8.0 + proc_thp_coverage: float = 0.6 + cpu_node_residency: dict[int, float] = field(default_factory=lambda: {0: 0.5, 1: 0.5}) + mem_by_node: dict[int, float] = field(default_factory=lambda: {0: 0.5, 1: 0.5}) + # hardware counters (None = not collected) + counters: dict[str, float] | None = None + + @property + def ncpu(self) -> int: + return self.sockets * self.cores_per_socket * self.smt + + +SCENARIOS: dict[str, tuple[str, Spec]] = { + "healthy": ( + "Two-socket server, balanced and unsaturated", + Spec( + counters={ + "cycles": 3e10, + "instructions": 6.6e10, + "cache-references": 4e8, + "cache-misses": 2e7, + "branches": 1.2e10, + "branch-misses": 6e7, + "dTLB-loads": 2e10, + "dTLB-load-misses": 4e7, + } + ), + ), + "numa_misplaced": ( + "Database whose heap was first-touched on node 1, while its threads run on node 0; THP misconfigured", + Spec( + comm="pgdb", + cpu_node_residency={0: 0.95, 1: 0.05}, + mem_by_node={0: 0.15, 1: 0.85}, + remote_alloc_frac=0.35, + hint_faults_per_s=40_000, + hint_local_frac=0.25, + node_free_frac=(0.45, 0.02), + proc_anon_gib=48, + proc_thp_coverage=0.04, + thp_enabled="always", + thp_defrag="always", + thp_alloc_per_s=50, + thp_fallback_per_s=40, + compact_stall_per_s=12, + hugetlb_2m_nr=2048, + hugetlb_2m_free=2048, + counters={ + "cycles": 4e10, + "instructions": 2.4e10, + "cache-references": 9e8, + "cache-misses": 4.3e8, + "branches": 4e9, + "branch-misses": 2e7, + "dTLB-loads": 8e9, + "dTLB-load-misses": 2.4e8, + }, + ), + ), + "cpu_saturated": ( + "Thread-pool oversubscription: 256 busy threads on 32 CPUs", + Spec( + comm="api-server", + threads=256, + cpu_util=0.98, + runq_waiting_tasks=24, + psi_cpu_some=0.85, + proc_cpus=30, + proc_runq_wait_share=0.45, + proc_invol_cs_per_s=9_000, + proc_vol_cs_per_s=3_000, + counters={"cycles": 9e10, "instructions": 1.1e11, "cache-references": 1.5e9, "cache-misses": 1.2e8, "branches": 2.2e10, "branch-misses": 6.6e8}, + ), + ), + "memory_pressure": ( + "Working set exceeds RAM: direct reclaim, swapping and major faults", + Spec( + comm="batch-etl", + psi_mem_some=0.35, + psi_mem_full=0.12, + allocstall_per_s=40, + swap_per_s=3_000, + major_faults_per_s=900, + proc_majflt_per_s=850, + proc_minflt_per_s=400_000, + node_free_frac=(0.01, 0.015), + ), + ), +} + + +def _cpu_line(label: str, busy: int, idle: int, steal: int = 0) -> str: + user, system = int(busy * 0.8), busy - int(busy * 0.8) + return f"{label} {user} 0 {system} {idle} 0 0 0 {steal} 0 0" + + +def _proc_stat(s: Spec, t: float) -> str: + total = int(t * USER_HZ) + lines = [] + all_busy = all_idle = all_steal = 0 + per = [] + for c in range(s.ncpu): + busy = int(total * s.cpu_util) + steal = int(total * s.steal) + idle = total - busy - steal + per.append(_cpu_line(f"cpu{c}", busy, idle, steal)) + all_busy, all_idle, all_steal = all_busy + busy, all_idle + idle, all_steal + steal + lines.append(_cpu_line("cpu", all_busy, all_idle, all_steal)) + lines += per + running = max(1, round(s.cpu_util * s.ncpu + s.runq_waiting_tasks)) + lines += ["intr 1000 0", f"ctxt {int(t * 50_000)}", "btime 1700000000", f"processes {int(t * 20)}", f"procs_running {running}", "procs_blocked 0"] + return "\n".join(lines) + "\n" + + +def _schedstat(s: Spec, t: float) -> str: + out = ["version 15", f"timestamp {int(t * 250)}"] + per_cpu_delay = s.runq_waiting_tasks / s.ncpu + for c in range(s.ncpu): + out.append(f"cpu{c} 0 0 0 0 0 0 {int(t * s.cpu_util * 1e9)} {int(t * per_cpu_delay * 1e9)} {int(t * 1000)}") + return "\n".join(out) + "\n" + + +def _vmstat(s: Spec, t: float) -> str: + v = { + "pgfault": s.faults_per_s * t, + "pgmajfault": s.major_faults_per_s * t, + "pswpin": s.swap_per_s * t / 2, + "pswpout": s.swap_per_s * t / 2, + "allocstall_normal": s.allocstall_per_s * t, + "allocstall_movable": 0, + "pgscan_direct": s.allocstall_per_s * t * 32, + "pgscan_kswapd": 0, + "oom_kill": 0, + "numa_hint_faults": s.hint_faults_per_s * t, + "numa_hint_faults_local": s.hint_faults_per_s * t * s.hint_local_frac, + "numa_pages_migrated": s.migrated_per_s * t, + "thp_fault_alloc": s.thp_alloc_per_s * t, + "thp_fault_fallback": s.thp_fallback_per_s * t, + "thp_collapse_alloc": 0, + "thp_split_page": 0, + "compact_stall": s.compact_stall_per_s * t, + "compact_fail": s.compact_stall_per_s * t * 0.5, + } + return "".join(f"{k} {int(1_000_000 + x)}\n" for k, x in v.items()) + + +def _meminfo(s: Spec, prefix: str = "", total_gib: float | None = None, free_frac: float | None = None) -> str: + total = int((total_gib or s.mem_per_node_gib * s.sockets) * (1 << 20)) + free = int(total * (free_frac if free_frac is not None else sum(s.node_free_frac) / len(s.node_free_frac))) + rows = { + "MemTotal": total, + "MemFree": free, + "MemAvailable": int(free * 1.1), + "AnonHugePages": 0, + "HugePages_Total": s.hugetlb_2m_nr, + "HugePages_Free": s.hugetlb_2m_free, + "Hugepagesize": 2048, + } + return "".join(f"{prefix}{k}: {v} kB\n" for k, v in rows.items()) + + +def _psi(some: float, full: float | None, t: float) -> str: + line = f"some avg10={some * 100:.2f} avg60={some * 100:.2f} avg300={some * 100:.2f} total={int(t * some * 1e6)}\n" + if full is not None: + line += f"full avg10={full * 100:.2f} avg60={full * 100:.2f} avg300={full * 100:.2f} total={int(t * full * 1e6)}\n" + return line + + +def _numastat(s: Spec, t: float, node: int) -> str: + allocs = s.allocs_per_s * t / s.sockets + other = allocs * s.remote_alloc_frac + return f"numa_hit {int(allocs)}\nnuma_miss 0\nnuma_foreign 0\ninterleave_hit 0\nlocal_node {int(allocs - other)}\nother_node {int(other)}\n" + + +def _pid_files(s: Spec, t: float, i: int) -> dict[str, str]: + assert s.pid + cpu_time = int(s.proc_cpus * t * USER_HZ) + # Deterministic CPU choice per sample following the residency weights. + nodes = sorted(s.cpu_node_residency) + acc, pick = 0.0, nodes[-1] + frac = (i * 0.618) % 1.0 + for n in nodes: + acc += s.cpu_node_residency[n] + if frac < acc: + pick = n + break + # CPU ids are numbered core-major (cpu = core + k * cores), so node n owns cores n*cores_per_socket... + processor = pick * s.cores_per_socket + (i % s.cores_per_socket) + rss_pages = int(s.proc_anon_gib * (1 << 18)) + stat_fields = [ + "S", + "1", + "4242", + "4242", + "0", + "-1", + "4194560", + str(int(s.proc_minflt_per_s * t)), + "0", + str(int(s.proc_majflt_per_s * t)), + "0", + str(int(cpu_time * 0.85)), + str(int(cpu_time * 0.15)), + "0", + "0", + "20", + "0", + str(s.threads), + "0", + "100", + "1000000", + str(rss_pages), + ] + stat_fields += ["18446744073709551615"] + ["0"] * 12 + ["17", str(processor)] + ["0"] * 13 # fields 25-52; exit_signal=38, processor=39 + on = s.proc_cpus * t * 1e9 + wait = on * s.proc_runq_wait_share / (1 - s.proc_runq_wait_share) + anon_kib = int(s.proc_anon_gib * (1 << 20)) + pages = anon_kib // 4 + huge = int(anon_kib * s.proc_thp_coverage) + numa_lines = [] + for n, f in sorted(s.mem_by_node.items()): + numa_lines.append(f"7f{n:02x}00000000 default anon={int(pages * f)} dirty={int(pages * f)} active=0 N{n}={int(pages * f)} kernelpagesize_kB=4") + return { + f"/proc/{s.pid}/stat": f"{s.pid} ({s.comm}) " + " ".join(stat_fields) + "\n", + f"/proc/{s.pid}/status": ( + f"Name:\t{s.comm}\nThreads:\t{s.threads}\nVmRSS:\t{anon_kib} kB\nCpus_allowed_list:\t0-{s.ncpu - 1}\n" + f"voluntary_ctxt_switches:\t{int(s.proc_vol_cs_per_s * t)}\nnonvoluntary_ctxt_switches:\t{int(s.proc_invol_cs_per_s * t)}\n" + ), + f"/proc/{s.pid}/schedstat": f"{int(on)} {int(wait)} {int(t * 5000)}\n", + f"/proc/{s.pid}/smaps_rollup": f"00400000-7fff00000000 ---p 00000000 00:00 0 [rollup]\nRss: {anon_kib} kB\nAnonymous: {anon_kib} kB\nAnonHugePages: {huge} kB\nSwap: 0 kB\nPrivate_Hugetlb: 0 kB\nShared_Hugetlb: 0 kB\n", + f"/proc/{s.pid}/numa_maps": "\n".join(numa_lines) + "\n", + } + + +def static_files(s: Spec) -> dict[str, str]: + files = { + "/proc/version": "Linux version 6.8.0-synthetic\n", + "/proc/sys/kernel/numa_balancing": s.numa_balancing + "\n", + "/proc/sys/kernel/perf_event_paranoid": "2\n", + "/proc/sys/vm/swappiness": "60\n", + "/proc/sys/vm/zone_reclaim_mode": s.zone_reclaim_mode + "\n", + "/sys/kernel/mm/transparent_hugepage/enabled": " ".join(f"[{m}]" if m == s.thp_enabled else m for m in ("always", "madvise", "never")) + "\n", + "/sys/kernel/mm/transparent_hugepage/defrag": " ".join( + f"[{m}]" if m == s.thp_defrag else m for m in ("always", "defer", "defer+madvise", "madvise", "never") + ) + + "\n", + "/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none": "511\n", + "/sys/devices/system/cpu/online": f"0-{s.ncpu - 1}\n", + "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages": f"{s.hugetlb_2m_nr}\n", + "/sys/kernel/mm/hugepages/hugepages-2048kB/free_hugepages": f"{s.hugetlb_2m_free}\n", + "/sys/kernel/mm/hugepages/hugepages-2048kB/resv_hugepages": "0\n", + "/sys/kernel/mm/hugepages/hugepages-2048kB/surplus_hugepages": "0\n", + } + cores = s.sockets * s.cores_per_socket + per_node = s.ncpu // s.sockets + for c in range(s.ncpu): + core = c % cores + sock = core // s.cores_per_socket + siblings = ",".join(str(core + k * cores) for k in range(s.smt)) + base = f"/sys/devices/system/cpu/cpu{c}" + files[f"{base}/topology/physical_package_id"] = f"{sock}\n" + files[f"{base}/topology/core_id"] = f"{core % s.cores_per_socket}\n" + files[f"{base}/topology/thread_siblings_list"] = siblings + "\n" + files[f"{base}/cpufreq/scaling_governor"] = "performance\n" + sock_cpus = ",".join(str(x) for x in range(s.ncpu) if (x % cores) // s.cores_per_socket == sock) + for idx, (lvl, typ, size, shared) in enumerate( + ((1, "Data", "48K", siblings), (1, "Instruction", "32K", siblings), (2, "Unified", "2048K", siblings), (3, "Unified", "32768K", sock_cpus)) + ): + cb = f"{base}/cache/index{idx}" + files.update( + { + f"{cb}/level": f"{lvl}\n", + f"{cb}/type": f"{typ}\n", + f"{cb}/size": f"{size}\n", + f"{cb}/shared_cpu_list": f"{shared}\n", + f"{cb}/coherency_line_size": "64\n", + } + ) + for n in range(s.sockets): + cpus = [x for x in range(s.ncpu) if (x % cores) // s.cores_per_socket == n] + files[f"/sys/devices/system/node/node{n}/cpulist"] = ",".join(map(str, cpus)) + "\n" + files[f"/sys/devices/system/node/node{n}/distance"] = " ".join("10" if m == n else "21" for m in range(s.sockets)) + "\n" + assert per_node > 0 + return files + + +def recording(spec: Spec) -> Recording: + n = int(spec.duration_s / spec.interval_s) + 1 + samples = [] + for i in range(n): + t = 1000.0 + i * spec.interval_s # counters have history before the window + files = { + "/proc/stat": _proc_stat(spec, t), + "/proc/schedstat": _schedstat(spec, t), + "/proc/vmstat": _vmstat(spec, t), + "/proc/meminfo": _meminfo(spec), + "/proc/pressure/cpu": _psi(spec.psi_cpu_some, None, t), + "/proc/pressure/memory": _psi(spec.psi_mem_some, spec.psi_mem_full, t), + } + for node in range(spec.sockets): + files[f"/sys/devices/system/node/node{node}/numastat"] = _numastat(spec, t, node) + files[f"/sys/devices/system/node/node{node}/meminfo"] = _meminfo(spec, f"Node {node} ", spec.mem_per_node_gib, spec.node_free_frac[node]) + if spec.pid: + files.update(_pid_files(spec, t, i)) + samples.append(Sample(1_700_000_000 + i * spec.interval_s, files, 0.0004)) + meta = { + "tool": "kptk synth", + "host": "synthetic-2s", + "kernel": "6.8.0-synthetic", + "arch": "x86_64", + "interval_s": spec.interval_s, + "pid": spec.pid, + "command": None, + "user_hz": USER_HZ, + "page_size": 4096, + "overruns": 0, + } + counters = {"events": {k: {"value": v, "raw": int(v), "running_frac": 1.0} for k, v in spec.counters.items()}, "skipped": {}} if spec.counters else None + return Recording(meta, static_files(spec), samples, counters) + + +def scenario(name: str, **overrides: object) -> Recording: + if name not in SCENARIOS: + raise ValueError(f"unknown scenario {name!r}; choose from {sorted(SCENARIOS)}") + return recording(replace(SCENARIOS[name][1], **overrides)) # type: ignore[arg-type] diff --git a/src/kptk/topology.py b/src/kptk/topology.py new file mode 100644 index 0000000..5e7e5e6 --- /dev/null +++ b/src/kptk/topology.py @@ -0,0 +1,148 @@ +"""CPU / cache / NUMA topology reconstructed from captured sysfs files.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from .parsers.sysfs import parse_cpulist, parse_distance, parse_size + +_CPU = re.compile(r"/sys/devices/system/cpu/cpu(\d+)/") +_NODE = re.compile(r"/sys/devices/system/node/node(\d+)/") + + +@dataclass(frozen=True) +class Cache: + level: int + type: str + size_bytes: int + shared_cpus: tuple[int, ...] + line_bytes: int = 64 + + +@dataclass +class Topology: + cpus: list[int] = field(default_factory=list) + cpu_node: dict[int, int] = field(default_factory=dict) + cpu_package: dict[int, int] = field(default_factory=dict) + cpu_core: dict[int, int] = field(default_factory=dict) + smt_siblings: dict[int, list[int]] = field(default_factory=dict) + node_cpus: dict[int, list[int]] = field(default_factory=dict) + distances: dict[int, list[int]] = field(default_factory=dict) + caches: list[Cache] = field(default_factory=list) + governors: dict[int, str] = field(default_factory=dict) + + @property + def nodes(self) -> list[int]: + return sorted(self.node_cpus) or [0] + + @property + def packages(self) -> int: + return len(set(self.cpu_package.values())) or 1 + + @property + def physical_cores(self) -> int: + return len(set(zip(self.cpu_package.values(), self.cpu_core.values(), strict=False))) or len(self.cpus) + + @property + def smt(self) -> bool: + return any(len(s) > 1 for s in self.smt_siblings.values()) + + def distance(self, a: int, b: int) -> int: + row = self.distances.get(a) + return row[b] if row and b < len(row) else (10 if a == b else 20) + + def cache(self, level: int, kind: str = "Unified") -> Cache | None: + for c in self.caches: + if c.level == level and (c.type == kind or (kind == "Data" and c.type == "Data")): + return c + return None + + @property + def llc(self) -> Cache | None: + return max(self.caches, key=lambda c: c.level, default=None) + + def summary(self) -> dict[str, object]: + return { + "cpus": len(self.cpus), + "packages": self.packages, + "physical_cores": self.physical_cores, + "smt": self.smt, + "numa_nodes": len(self.nodes), + "node_cpus": {n: _fmt_cpus(c) for n, c in self.node_cpus.items()}, + "distances": self.distances, + "caches": [ + {"level": c.level, "type": c.type, "size_kib": c.size_bytes // 1024, "shared_by_cpus": len(c.shared_cpus), "instances": self._instances(c)} + for c in self.caches + ], + "governors": sorted(set(self.governors.values())), + } + + def _instances(self, c: Cache) -> int: + return max(1, len(self.cpus) // max(1, len(c.shared_cpus))) + + +def _fmt_cpus(cpus: list[int]) -> str: + if not cpus: + return "" + out, start, prev = [], cpus[0], cpus[0] + for c in [*cpus[1:], None]: + if c is not None and c == prev + 1: + prev = c + continue + out.append(f"{start}-{prev}" if prev != start else str(start)) + if c is not None: + start = prev = c + return ",".join(out) + + +def from_static(files: dict[str, str]) -> Topology: + topo = Topology() + online = files.get("/sys/devices/system/cpu/online") + cpus: set[int] = set(parse_cpulist(online)) if online else set() + cache_parts: dict[tuple[int, str], dict[str, str]] = {} + for path, text in files.items(): + if m := _CPU.match(path): + cpu = int(m.group(1)) + cpus.add(cpu) + value = text.strip() + if path.endswith("/topology/physical_package_id"): + topo.cpu_package[cpu] = int(value) + elif path.endswith("/topology/core_id"): + topo.cpu_core[cpu] = int(value) + elif path.endswith("/topology/thread_siblings_list"): + topo.smt_siblings[cpu] = parse_cpulist(value) + elif path.endswith("/cpufreq/scaling_governor"): + topo.governors[cpu] = value + elif "/cache/index" in path: + idx = path.split("/cache/")[1].split("/")[0] + cache_parts.setdefault((cpu, idx), {})[path.rsplit("/", 1)[1]] = value + elif m := _NODE.match(path): + node = int(m.group(1)) + if path.endswith("/cpulist"): + topo.node_cpus[node] = parse_cpulist(text) + for c in topo.node_cpus[node]: + topo.cpu_node[c] = node + elif path.endswith("/distance"): + topo.distances[node] = parse_distance(text) + topo.cpus = sorted(cpus) + for c in topo.cpus: + topo.cpu_node.setdefault(c, 0) + if not topo.node_cpus: + topo.node_cpus = {0: topo.cpus} + seen: set[tuple[int, str, tuple[int, ...]]] = set() + for parts in cache_parts.values(): + if not {"level", "type", "size"} <= parts.keys(): + continue + shared = tuple(parse_cpulist(parts.get("shared_cpu_list", ""))) + key = (int(parts["level"]), parts["type"], shared) + if key in seen: + continue + seen.add(key) + topo.caches.append(Cache(key[0], key[1], parse_size(parts["size"]), shared, int(parts.get("coherency_line_size", "64") or 64))) + # Keep one representative per (level, type); sizes are per instance. + unique: dict[tuple[int, str], Cache] = {} + for cache in sorted(topo.caches, key=lambda x: (x.level, x.type, -len(x.shared_cpus))): + unique.setdefault((cache.level, cache.type), cache) + topo.caches = sorted(unique.values(), key=lambda c: (c.level, c.type)) + return topo diff --git a/src/numa_analysis.py b/src/numa_analysis.py deleted file mode 100644 index dc5a300..0000000 --- a/src/numa_analysis.py +++ /dev/null @@ -1,11 +0,0 @@ -"""NUMA balance calculations.""" - -from __future__ import annotations - - -def numa_metrics(numastat: dict[str, dict[str, float]]) -> dict[str, float]: - local = numastat.get("numa_hit", {}).get("total", 0.0) - remote = numastat.get("numa_miss", {}).get("total", 0.0) - total = local + remote - return {"numa_local_allocations": local, "numa_remote_allocations": remote, - "numa_remote_percent": 100 * remote / total if total else 0.0} diff --git a/src/page_faults.py b/src/page_faults.py deleted file mode 100644 index c09fd1c..0000000 --- a/src/page_faults.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Page-fault metrics.""" - -from __future__ import annotations - - -def page_fault_metrics(perf: dict[str, float]) -> dict[str, float]: - minor = perf.get("minor-faults", perf.get("page-faults", 0.0)) - major = perf.get("major-faults", 0.0) - total = minor + major - return {"minor_faults": minor, "major_faults": major, "total_faults": total, - "major_fault_percent": 100 * major / total if total else 0.0} diff --git a/src/parser.py b/src/parser.py deleted file mode 100644 index 5a25bc6..0000000 --- a/src/parser.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Parsers for Linux performance command output.""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Iterable - - -_NUMBER = re.compile(r"^([0-9][0-9,]*(?:\.[0-9]+)?)([KMG]?)$") - - -def _number(value: str) -> float: - """Convert perf-style numbers (including commas and K/M/G suffixes).""" - match = _NUMBER.match(value.strip()) - if not match: - raise ValueError(f"invalid numeric value: {value!r}") - amount = float(match.group(1).replace(",", "")) - return amount * {"": 1, "K": 1_000, "M": 1_000_000, "G": 1_000_000_000}[match.group(2)] - - -def parse_perf_stat(text: str) -> dict[str, float]: - """Parse ``perf stat`` CSV or human-readable output into event counters. - - Unsupported, unavailable counters (````) are deliberately - ignored so reports remain useful on restricted hosts. - """ - metrics: dict[str, float] = {} - for raw_line in text.splitlines(): - line = raw_line.strip() - if not line or line.startswith("#") or "= 3 and fields[0] and fields[2]: - try: - metrics[fields[2]] = _number(fields[0]) - except ValueError: - pass - continue - fields = line.split() - if len(fields) < 2: - continue - try: - value = _number(fields[0]) - except ValueError: - continue - # perf may print a unit between the count and event name. - event_index = 2 if len(fields) > 2 and fields[1] in {"msec", "seconds", "ns"} else 1 - if event_index < len(fields): - metrics[fields[event_index]] = value - return metrics - - -def parse_vmstat(text: str) -> dict[str, float]: - """Parse one or more ``vmstat`` snapshots, retaining the last sample.""" - headers: list[str] | None = None - latest: dict[str, float] = {} - for raw_line in text.splitlines(): - fields = raw_line.split() - if not fields: - continue - if fields[0] == "procs": - headers = None - continue - if fields[0] == "r" and "free" in fields: - headers = fields - continue - if headers and len(fields) >= len(headers) and all(item.lstrip("-").isdigit() for item in fields[:len(headers)]): - latest = {key: float(value) for key, value in zip(headers, fields)} - return latest - - -def parse_numastat(text: str) -> dict[str, dict[str, float]]: - """Parse ``numastat`` output as metric -> node -> value.""" - result: dict[str, dict[str, float]] = {} - nodes: list[str] = [] - for raw_line in text.splitlines(): - fields = raw_line.split() - if fields and all(re.fullmatch(r"node\d+", field.lower()) for field in fields): - nodes = [field.lower() for field in fields] - continue - if len(fields) < 2 or fields[0].lower() in {"node", "per-node"}: - continue - metric = fields[0] - values = fields[1:] - if not all(re.fullmatch(r"[0-9.,]+", value) for value in values): - continue - numeric_values = [float(value.replace(",", "")) for value in values] - if nodes and len(numeric_values) == len(nodes): - result[metric] = dict(zip(nodes, numeric_values)) - result[metric]["total"] = sum(numeric_values) - elif len(numeric_values) >= 2: # output with a trailing Total column - result[metric] = {f"node{i}": value for i, value in enumerate(numeric_values[:-1])} - result[metric]["total"] = numeric_values[-1] - return result - - -def read_and_parse(path: str | Path, parser: object) -> dict: - return parser(Path(path).read_text(encoding="utf-8")) diff --git a/src/perf_runner.py b/src/perf_runner.py deleted file mode 100644 index 0daf060..0000000 --- a/src/perf_runner.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -"""Collect or parse Linux performance data and emit a JSON report.""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - -from parser import parse_numastat, parse_perf_stat, parse_vmstat -from report import build_report, markdown_summary, write_json -from utils import run - -DEFAULT_EVENTS = "cycles,instructions,cache-references,cache-misses,context-switches,cpu-migrations,page-faults,minor-faults,major-faults,task-clock" - - -def collect(command: list[str], events: str) -> tuple[dict[str, float], dict[str, float], dict[str, dict[str, float]]]: - perf_output = run(["perf", "stat", "-x", ";", "-e", events, "--", *command]) - return parse_perf_stat(perf_output), parse_vmstat(run(["vmstat", "1", "2"])), parse_numastat(run(["numastat"])) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - source = parser.add_mutually_exclusive_group(required=True) - source.add_argument("--perf-file", type=Path, help="parse saved perf stat output") - source.add_argument("--command", nargs=argparse.REMAINDER, help="run a command under perf stat; use after --command") - parser.add_argument("--vmstat-file", type=Path, help="optional saved vmstat output") - parser.add_argument("--numastat-file", type=Path, help="optional saved numastat output") - parser.add_argument("--events", default=DEFAULT_EVENTS, help="comma-separated perf events") - parser.add_argument("--output", type=Path, default=Path("report.json"), help="JSON output path") - parser.add_argument("--markdown", type=Path, help="optional Markdown summary path") - args = parser.parse_args(argv) - - try: - if args.perf_file: - perf = parse_perf_stat(args.perf_file.read_text(encoding="utf-8")) - vmstat = parse_vmstat(args.vmstat_file.read_text(encoding="utf-8")) if args.vmstat_file else {} - numa = parse_numastat(args.numastat_file.read_text(encoding="utf-8")) if args.numastat_file else {} - else: - if not args.command: - parser.error("--command requires a command, for example: --command sleep 1") - perf, vmstat, numa = collect(args.command, args.events) - report = build_report(perf, vmstat, numa) - write_json(report, args.output) - if args.markdown: - args.markdown.write_text(markdown_summary(report), encoding="utf-8") - except (OSError, RuntimeError, ValueError) as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/report.py b/src/report.py deleted file mode 100644 index 80b378b..0000000 --- a/src/report.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Report creation and serialization.""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - - -def build_report(perf: dict[str, float], vmstat: dict[str, float] | None = None, - numa: dict[str, dict[str, float]] | None = None) -> dict[str, Any]: - from cache_analysis import cache_metrics - from numa_analysis import numa_metrics - from page_faults import page_fault_metrics - from scheduler_latency import scheduler_metrics - - report: dict[str, Any] = { - "generated_at": datetime.now(timezone.utc).isoformat(), - "perf": perf, - "metrics": {**cache_metrics(perf), **page_fault_metrics(perf), **scheduler_metrics(perf)}, - } - if vmstat: - report["vmstat"] = vmstat - if numa: - report["numastat"] = numa - report["metrics"].update(numa_metrics(numa)) - return report - - -def write_json(report: dict[str, Any], output: str | Path) -> None: - Path(output).write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def markdown_summary(report: dict[str, Any]) -> str: - metrics = report["metrics"] - lines = ["# Kernel Performance Report", "", "| Metric | Value |", "| --- | ---: |"] - for name, value in sorted(metrics.items()): - lines.append(f"| {name.replace('_', ' ')} | {value:.2f} |") - return "\n".join(lines) + "\n" diff --git a/src/scheduler_latency.py b/src/scheduler_latency.py deleted file mode 100644 index 14ec459..0000000 --- a/src/scheduler_latency.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Scheduler-related perf calculations.""" - -from __future__ import annotations - - -def scheduler_metrics(perf: dict[str, float]) -> dict[str, float]: - context_switches = perf.get("context-switches", 0.0) - migrations = perf.get("cpu-migrations", 0.0) - task_clock = perf.get("task-clock", 0.0) - return {"context_switches": context_switches, "cpu_migrations": migrations, - "context_switches_per_ms": context_switches / task_clock if task_clock else 0.0} diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index a4e5896..0000000 --- a/src/utils.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Shared command helpers.""" - -from __future__ import annotations - -import subprocess - - -def run(command: list[str], timeout: int = 60) -> str: - """Run a command and return stdout, raising a helpful exception on failure.""" - try: - result = subprocess.run(command, check=True, text=True, capture_output=True, timeout=timeout) - except FileNotFoundError as exc: - raise RuntimeError(f"command not found: {command[0]}") from exc - except subprocess.CalledProcessError as exc: - detail = exc.stderr.strip() or exc.stdout.strip() - raise RuntimeError(f"{' '.join(command)} failed: {detail}") from exc - # ``perf stat`` writes measurements to stderr; returning both streams also - # preserves tools which use stderr for informational output. - return result.stdout + result.stderr diff --git a/tests/test_analysis.py b/tests/test_analysis.py new file mode 100644 index 0000000..171ffb1 --- /dev/null +++ b/tests/test_analysis.py @@ -0,0 +1,111 @@ +import json + +import pytest + +from kptk.analysis.engine import analyze +from kptk.analysis.findings import Thresholds +from kptk.compare import compare, permutation_p +from kptk.synth import scenario + +EXPECTED = { + "healthy": set(), + "numa_misplaced": { + "numa.process_misplaced", + "numa.remote_alloc", + "numa.access_locality", + "numa.node_exhausted", + "cache.memory_bound", + "cache.tlb_pressure", + "hp.sync_compaction", + "hp.thp_fallback", + "hp.compaction_stalls", + "hp.low_coverage", + }, + "cpu_saturated": {"cpu.saturation", "cpu.process_runq_wait", "cpu.preemption"}, + "memory_pressure": { + "mem.swapping", + "mem.pressure", + "mem.direct_reclaim", + "mem.low_available", + "mem.major_faults", + "mem.minor_fault_storm", + "numa.node_exhausted", + }, +} + + +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_scenario_findings_are_exact(name: str) -> None: + report = analyze(scenario(name)) + actionable = {f["id"] for f in report["findings"] if f["severity"] != "info"} + assert actionable == EXPECTED[name] + json.dumps(report, default=str) + + +def test_metrics_recover_synthetic_ground_truth() -> None: + m = analyze(scenario("cpu_saturated"))["metrics"] + assert m["cpu"]["utilization"] == pytest.approx(0.98, abs=0.01) + assert m["cpu"]["psi_cpu_some"] == pytest.approx(0.85, abs=0.01) + assert m["cpu"]["runq_waiting_tasks_avg"] == pytest.approx(24, rel=0.01) + assert m["cpu"]["process_runq_wait_share"] == pytest.approx(0.45, abs=0.01) + assert m["cpu"]["process_cpus_used"] == pytest.approx(30, rel=0.01) + n = analyze(scenario("numa_misplaced"))["metrics"] + assert n["numa"]["remote_alloc_frac"] == pytest.approx(0.35, abs=0.01) + assert n["numa"]["hint_fault_local_frac"] == pytest.approx(0.25, abs=0.01) + assert n["numa"]["process_placement_mismatch"] == pytest.approx(0.85, abs=0.01) + assert n["hugepages"]["thp_fallback_frac"] == pytest.approx(40 / 90, abs=0.01) + assert n["cache"]["ipc"] == pytest.approx(0.6) and n["cache"]["llc_mpki"] == pytest.approx(17.92, abs=0.01) + + +def test_spread_process_with_spread_memory_is_not_misplaced() -> None: + # 50/50 threads and 50/50 memory: ~50% remote under uniform access, but not a placement bug. + m = analyze(scenario("healthy"))["metrics"]["numa"] + assert m["process_remote_access_frac_est"] == pytest.approx(0.5, abs=0.1) + assert m["process_placement_mismatch"] < 0.1 + + +def test_thresholds_override(tmp_path) -> None: + p = tmp_path / "t.json" + p.write_text(json.dumps({"cpu_psi_some_warn": 0.9, "runq_wait_share_warn": 0.9, "involuntary_switch_rate_warn": 1e9})) + report = analyze(scenario("cpu_saturated"), thresholds=Thresholds.load(p)) + assert not [f for f in report["findings"] if f["domain"] == "cpu" and f["severity"] != "info"] + p.write_text(json.dumps({"bogus": 1})) + with pytest.raises(ValueError): + Thresholds.load(p) + + +def test_missing_schedstat_is_reported_not_fatal() -> None: + rec = scenario("healthy") + for s in rec.samples: + s.files.pop("/proc/schedstat") + report = analyze(rec) + assert report["metrics"]["cpu"]["runq_waiting_tasks_avg"] is None + assert any("SCHEDSTATS" in x for x in report["data_quality"]["missing"]) + + +def test_needs_two_samples() -> None: + rec = scenario("healthy") + rec.samples = rec.samples[:1] + with pytest.raises(ValueError): + analyze(rec) + + +def test_permutation_test() -> None: + assert permutation_p([1.0, 1.1, 0.9, 1.0], [2.0, 2.1, 1.9, 2.0]) < 0.05 + assert permutation_p([1.0, 2.0, 1.5], [1.1, 1.9, 1.4]) > 0.5 + + +def test_compare_flags_regression_with_significance() -> None: + before = [analyze(scenario("healthy", proc_runq_wait_share=x)) for x in (0.02, 0.021, 0.019, 0.02)] + after = [analyze(scenario("healthy", proc_runq_wait_share=x)) for x in (0.08, 0.081, 0.079, 0.08)] + changes = {c.metric: c for c in compare(before, after)} + c = changes["cpu.process_runq_wait_share"] + assert c.verdict == "regressed" and c.p_value is not None and c.p_value < 0.05 + assert changes["cpu.utilization"].verdict == "no significant change" + + +def test_compare_higher_is_better() -> None: + before = [analyze(scenario("numa_misplaced"))] + after = [analyze(scenario("numa_misplaced", hint_local_frac=0.9))] + c = {c.metric: c for c in compare(before, after)}["numa.hint_fault_local_frac"] + assert c.verdict.startswith("improved") and c.p_value is None diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..1a5f79a --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,76 @@ +import json +import sys + +from kptk.cli import main +from kptk.synth import scenario + + +def test_demo_and_list(capsys) -> None: + assert main(["demo", "--list"]) == 0 + assert "numa_misplaced" in capsys.readouterr().out + assert main(["demo", "cpu_saturated"]) == 0 + assert "CPU saturation" in capsys.readouterr().out + + +def test_analyze_recording_with_gate(tmp_path, capsys) -> None: + path = tmp_path / "rec.jsonl" + scenario("memory_pressure").save(path) + out = tmp_path / "r.json" + assert main(["analyze", str(path), "--json", str(out), "--md", str(tmp_path / "r.md"), "--fail-on", "critical"]) == 3 + assert json.loads(out.read_text())["findings"][0]["severity"] == "critical" + assert main(["analyze", str(path), "--md", str(tmp_path / "r2.md")]) == 0 + + +def test_analyze_with_perf_stat_file(tmp_path, capsys) -> None: + path = tmp_path / "rec.jsonl" + scenario("healthy", counters=None).save(path) + perf = tmp_path / "perf.txt" + perf.write_text("1000;;cycles;1;100.00;;\n400;;instructions;1;100.00;;\n100;;cache-references;1;100.00;;\n30;;cache-misses;1;100.00;;\n") + assert main(["analyze", str(path), "--perf-stat", str(perf)]) == 0 + assert "memory-bound" in capsys.readouterr().out + + +def test_inspect_json(capsys) -> None: + assert main(["inspect", "--json"]) == 0 + doc = json.loads(capsys.readouterr().out) + assert doc["topology"]["cpus"] >= 1 + + +def test_parse_legacy_samples(tmp_path) -> None: + out = tmp_path / "o.json" + assert ( + main( + [ + "parse", + "--perf-file", + "samples/perf_output.txt", + "--vmstat-file", + "samples/vmstat.txt", + "--numastat-file", + "samples/numastat.txt", + "-o", + str(out), + ] + ) + == 0 + ) + doc = json.loads(out.read_text()) + assert doc["cache"]["ipc"] == 2.5 + assert doc["policy_miss_frac"] > 0 + + +def test_compare_cli(tmp_path, capsys) -> None: + from kptk.analysis.engine import analyze + + a, b = tmp_path / "a.json", tmp_path / "b.json" + a.write_text(json.dumps(analyze(scenario("healthy")), default=str)) + b.write_text(json.dumps(analyze(scenario("cpu_saturated")), default=str)) + assert main(["compare", "--before", str(a), "--after", str(b), "--fail-on-regression"]) == 3 + assert "n=1, untested" in capsys.readouterr().out + + +def test_profile_short_command(tmp_path, capsys) -> None: + rec = tmp_path / "p.jsonl" + rc = main(["profile", "-i", "0.05", "-o", str(rec), "--md", str(tmp_path / "p.md"), "--", sys.executable, "-c", "import time; time.sleep(0.3)"]) + assert rc == 0 + assert (tmp_path / "p.md").read_text().startswith("# Kernel performance report") diff --git a/tests/test_metrics.py b/tests/test_metrics.py deleted file mode 100644 index 4b0f69e..0000000 --- a/tests/test_metrics.py +++ /dev/null @@ -1,17 +0,0 @@ -from pathlib import Path -import sys - -sys.path.insert(0, str(Path(__file__).parents[1] / "src")) - -from cache_analysis import cache_metrics -from numa_analysis import numa_metrics -from page_faults import page_fault_metrics - - -def test_cache_miss_rate(): - assert cache_metrics({"cache-references": 200, "cache-misses": 10})["cache_miss_rate_percent"] == 5 - - -def test_faults_and_numa_metrics_handle_counts(): - assert page_fault_metrics({"minor-faults": 90, "major-faults": 10})["major_fault_percent"] == 10 - assert numa_metrics({"numa_hit": {"total": 80}, "numa_miss": {"total": 20}})["numa_remote_percent"] == 20 diff --git a/tests/test_parser.py b/tests/test_parser.py deleted file mode 100644 index 9629a94..0000000 --- a/tests/test_parser.py +++ /dev/null @@ -1,26 +0,0 @@ -from pathlib import Path -import sys - -sys.path.insert(0, str(Path(__file__).parents[1] / "src")) - -from parser import parse_numastat, parse_perf_stat, parse_vmstat - - -def test_parse_perf_stat_csv_and_human_formats(): - data = "1,234; ;cycles; ;\n45; ;cache-misses; ;\n 10.0 msec task-clock\n" - assert parse_perf_stat(data) == {"cycles": 1234.0, "cache-misses": 45.0, "task-clock": 10.0} - - -def test_parse_vmstat_uses_last_snapshot(): - data = "procs -----------memory----------\nr b swpd free\n1 0 0 99\n2 0 0 88\n" - assert parse_vmstat(data)["free"] == 88.0 - - -def test_parse_numastat(): - result = parse_numastat("numa_hit 100 50 150\nnuma_miss 5 10 15\n") - assert result["numa_miss"] == {"node0": 5.0, "node1": 10.0, "total": 15.0} - - -def test_parse_numastat_without_total_column(): - result = parse_numastat("node0 node1\nnuma_hit 100 50\n") - assert result["numa_hit"]["total"] == 150.0 diff --git a/tests/test_parsers.py b/tests/test_parsers.py new file mode 100644 index 0000000..85185f3 --- /dev/null +++ b/tests/test_parsers.py @@ -0,0 +1,131 @@ +import pytest + +from kptk.parsers.proc import ( + busy_ticks, + parse_meminfo, + parse_numa_maps, + parse_pid_stat, + parse_pid_status, + parse_pressure, + parse_proc_stat, + parse_schedstat, + parse_smaps_rollup, + parse_task_schedstat, + parse_vmstat, + total_ticks, +) +from kptk.parsers.sysfs import parse_bracket_choice, parse_cpulist, parse_distance, parse_size +from kptk.parsers.tools import parse_numastat_output, parse_perf_stat, parse_vmstat_output + +PROC_STAT = """cpu 100 5 50 800 20 0 5 10 7 0 +cpu0 50 2 25 400 10 0 3 5 7 0 +cpu1 50 3 25 400 10 0 2 5 0 0 +intr 12345 0 0 +ctxt 99999 +btime 1700000000 +processes 4242 +procs_running 3 +procs_blocked 1 +""" + + +def test_proc_stat() -> None: + s = parse_proc_stat(PROC_STAT) + assert s.ctxt == 99999 and s.procs_running == 3 and s.procs_blocked == 1 and s.intr == 12345 + assert set(s.cpus) == {0, 1} + # guest time is already inside user, so it must not be double counted + assert busy_ticks(s.cpu_total) == 100 + 5 + 50 + 0 + 5 + 10 + assert total_ticks(s.cpu_total) == 990 + + +def test_schedstat_v15() -> None: + text = "version 15\ntimestamp 4708\ncpu0 0 0 0 0 0 0 1000 250 40\ndomain0 ff 1 2 3\ncpu1 0 0 0 0 0 0 2000 750 60\n" + version, cpus = parse_schedstat(text) + assert version == 15 + assert cpus[1].run_delay_ns == 750 and cpus[0].timeslices == 40 + + +def test_task_schedstat() -> None: + t = parse_task_schedstat("30170206 1127 64\n") + assert (t.on_cpu_ns, t.wait_ns, t.timeslices) == (30170206, 1127, 64) + + +def test_pid_stat_handles_comm_with_spaces_and_parens() -> None: + rest = ( + ["S", "1", "1", "1", "0", "-1", "0", "111", "0", "22", "0", "300", "40", "0", "0", "20", "0", "7", "0", "100", "1000", "555"] + + ["0"] * 13 + + ["17", "5"] + + ["0"] * 13 + ) + p = parse_pid_stat("1234 (my (weird) proc) " + " ".join(rest)) + assert p.comm == "my (weird) proc" + assert (p.minflt, p.majflt, p.utime, p.stime, p.num_threads, p.rss_pages, p.processor) == (111, 22, 300, 40, 7, 555, 5) + + +def test_pid_status() -> None: + s = parse_pid_status("Name:\tredis\nVmRSS:\t 1024 kB\nvoluntary_ctxt_switches:\t10\nCpus_allowed_list:\t0-3\n") + assert s["VmRSS"] == 1024 and s["voluntary_ctxt_switches"] == 10 and s["Cpus_allowed_list"] == "0-3" + + +def test_vmstat_and_meminfo() -> None: + assert parse_vmstat("pgfault 10\nnr_free_pages 5\nbogus\n") == {"pgfault": 10, "nr_free_pages": 5} + mi = parse_meminfo("MemTotal: 100 kB\nHugePages_Total: 4\n") + assert mi == {"MemTotal": 100, "HugePages_Total": 4} + assert parse_meminfo("Node 1 MemFree: 42 kB\n") == {"MemFree": 42} + + +def test_smaps_rollup_skips_range_header() -> None: + s = parse_smaps_rollup("00400000-7ffd00000000 ---p 00000000 00:00 0 [rollup]\nRss: 10 kB\nAnonHugePages: 4096 kB\n") + assert s == {"Rss": 10, "AnonHugePages": 4096} + + +def test_pressure() -> None: + p = parse_pressure("some avg10=1.50 avg60=0.50 avg300=0.10 total=1000\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=5\n") + assert p["some"].avg10 == 1.5 and p["full"].total_us == 5 + + +def test_numa_maps_normalises_huge_pages() -> None: + text = ( + "7f0000000000 default anon=100 dirty=100 N0=60 N1=40 kernelpagesize_kB=4\n" + "7f1000000000 bind:1 anon=2 dirty=2 N1=2 kernelpagesize_kB=2048\n" + "7f2000000000 interleave:0-1 file=/lib/x.so mapped=3 N0=3 kernelpagesize_kB=4\n" + ) + m = parse_numa_maps(text) + assert m.pages_by_node == {0: 63, 1: 40 + 2 * 512} + assert m.huge_pages_by_node == {1: 2} + assert m.policies == {"default": 1, "bind": 1, "interleave": 1} + assert m.anon_pages == 102 + + +def test_sysfs_helpers() -> None: + assert parse_cpulist("0-3,8,10-11\n") == [0, 1, 2, 3, 8, 10, 11] + assert parse_cpulist("") == [] + assert parse_bracket_choice("always [madvise] never\n") == "madvise" + assert parse_bracket_choice("none here") is None + assert parse_size("48K") == 48 * 1024 and parse_size("32M") == 32 << 20 + assert parse_distance("10 21\n") == [10, 21] + + +def test_perf_stat_csv_keeps_running_pct() -> None: + text = "1,234;;cycles;5000;50.00;;\n45;;cache-misses;5000;100.00;;\n;;LLC-loads;0;0.00;;\n" + out = parse_perf_stat(text) + assert out["cycles"].value == 1234 and out["cycles"].running_pct == 50.0 + assert "LLC-loads" not in out + + +def test_perf_stat_human_format() -> None: + text = " 10.00 msec task-clock\n 1,000,000 cycles (66.67%)\n 2.5G instructions\n" + out = parse_perf_stat(text) + assert out["task-clock"].value == 10.0 + assert out["cycles"].running_pct == pytest.approx(66.67) + assert out["instructions"].value == 2.5e9 + + +def test_vmstat_command_output() -> None: + text = "procs ---memory---\n r b swpd free\n 1 0 0 99\n 2 0 0 88\n" + assert parse_vmstat_output(text)["free"] == 88.0 + + +def test_numastat_command_output() -> None: + assert parse_numastat_output("numa_hit 100 50 150\n")["numa_hit"] == {"node0": 100.0, "node1": 50.0, "total": 150.0} + assert parse_numastat_output("node0 node1\nother_node 1 2\n")["other_node"]["total"] == 3.0 diff --git a/tests/test_perf_event.py b/tests/test_perf_event.py new file mode 100644 index 0000000..2924ba1 --- /dev/null +++ b/tests/test_perf_event.py @@ -0,0 +1,64 @@ +import ctypes + +import pytest + +from kptk.perf_event import ( + FLAG_DISABLED, + FLAG_ENABLE_ON_EXEC, + FLAG_INHERIT, + FORMAT_TOTAL_TIME_ENABLED, + FORMAT_TOTAL_TIME_RUNNING, + PERF_ATTR_SIZE_VER5, + Counters, + PerfEventAttr, + PerfUnavailable, + event_config, + make_attr, + scale, +) + + +def test_attr_layout_matches_uapi_header() -> None: + assert ctypes.sizeof(PerfEventAttr) == PERF_ATTR_SIZE_VER5 == 112 + offsets = {name: getattr(PerfEventAttr, name).offset for name, _ in PerfEventAttr._fields_} + assert offsets["config"] == 8 + assert offsets["read_format"] == 32 + assert offsets["flags"] == 40 + assert offsets["config1"] == 56 + assert offsets["sample_regs_intr"] == 96 + assert offsets["sample_max_stack"] == 108 + + +def test_event_encoding() -> None: + assert event_config("instructions") == (0, 1) + assert event_config("major-faults") == (1, 6) + # LLC (2) | read (0 << 8) | miss (1 << 16) + assert event_config("LLC-load-misses") == (3, 2 | (1 << 16)) + assert event_config("dTLB-loads") == (3, 3) + with pytest.raises(ValueError): + event_config("not-an-event") + + +def test_make_attr_flags() -> None: + a = make_attr("cycles", inherit=True, enable_on_exec=True) + assert a.flags & FLAG_DISABLED and a.flags & FLAG_INHERIT and a.flags & FLAG_ENABLE_ON_EXEC + assert a.read_format == FORMAT_TOTAL_TIME_ENABLED | FORMAT_TOTAL_TIME_RUNNING + assert a.size == 112 + + +def test_multiplex_scaling() -> None: + assert scale(1000, enabled=100, running=25) == (4000.0, 0.25) + assert scale(1000, 100, 0) == (0.0, 0.0) + + +def test_live_counters_or_clean_failure() -> None: + try: + with Counters(["task-clock", "instructions"], inherit=False) as c: + c.enable() + sum(range(1000)) + c.disable() + readings = c.read() + except PerfUnavailable as exc: + assert "perf_event_open failed" in str(exc) + pytest.skip(f"perf_event_open not permitted here: {exc}") + assert readings and readings[0].value > 0 diff --git a/tests/test_record_topology.py b/tests/test_record_topology.py new file mode 100644 index 0000000..cc0561e --- /dev/null +++ b/tests/test_record_topology.py @@ -0,0 +1,57 @@ +from kptk.record import DictSource, Recorder, Recording, expand +from kptk.synth import Spec, scenario, static_files +from kptk.topology import from_static + + +def test_dict_source_glob_and_slow_files() -> None: + files = { + "/proc/stat": "cpu 1 0 0 1 0 0 0 0 0 0\n", + "/sys/devices/system/node/node0/numastat": "local_node 1\n", + "/sys/devices/system/node/node1/numastat": "local_node 2\n", + "/proc/7/stat": "x", + "/proc/7/numa_maps": "y", + } + src = DictSource(files) + assert expand(src, ["/sys/devices/system/node/node[0-9]*/numastat"]) == [ + "/sys/devices/system/node/node0/numastat", + "/sys/devices/system/node/node1/numastat", + ] + rec = Recorder(src, interval_s=0.001, pid=7, slow_every=3).run(max_samples=4) + has_slow = ["/proc/7/numa_maps" in s.files for s in rec.samples] + assert has_slow == [True, False, False, True] + assert all("/proc/7/stat" in s.files for s in rec.samples) + assert rec.latest("/proc/7/numa_maps") == "y" + + +def test_recording_round_trip_gz(tmp_path) -> None: + rec = scenario("healthy") + path = tmp_path / "r.jsonl.gz" + rec.save(path) + back = Recording.load(path) + assert back.meta["pid"] == 4242 + assert back.static == rec.static + assert [s.files for s in back.samples] == [s.files for s in rec.samples] + assert back.counters == rec.counters + + +def test_live_recording_is_cheap_and_complete() -> None: + rec = Recorder(interval_s=0.05).run(max_samples=3) + assert len(rec.samples) == 3 + assert "/proc/stat" in rec.samples[0].files and "/proc/vmstat" in rec.samples[0].files + assert "/sys/devices/system/cpu/online" in rec.static + assert max(s.cost_s for s in rec.samples) < 0.5 + + +def test_topology_from_synthetic_two_socket() -> None: + topo = from_static(static_files(Spec())) + assert len(topo.cpus) == 32 and topo.nodes == [0, 1] and topo.packages == 2 + assert topo.physical_cores == 16 and topo.smt + assert topo.distance(0, 1) == 21 and topo.distance(1, 1) == 10 + assert topo.llc is not None and topo.llc.level == 3 and topo.llc.size_bytes == 32 << 20 + assert len(topo.llc.shared_cpus) == 16 + assert topo.summary()["node_cpus"][0] == "0-7,16-23" + + +def test_topology_live_smoke() -> None: + topo = from_static(Recorder().capture_static()) + assert topo.cpus and topo.nodes diff --git a/tests/test_reports.py b/tests/test_reports.py deleted file mode 100644 index 9864f17..0000000 --- a/tests/test_reports.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -import sys - -sys.path.insert(0, str(Path(__file__).parents[1] / "src")) - -from report import build_report, markdown_summary, write_json - - -def test_report_can_be_serialized(tmp_path): - report = build_report({"cache-references": 100, "cache-misses": 5}) - output = tmp_path / "report.json" - write_json(report, output) - assert '"cache_miss_rate_percent": 5.0' in output.read_text() - assert "Kernel Performance Report" in markdown_summary(report)