From 91ba24200a78d62d978d7ee3db0b4f65e7a5fa64 Mon Sep 17 00:00:00 2001 From: Noor Mkdad Date: Thu, 24 Sep 2026 00:31:06 -0700 Subject: [PATCH] Add a flake so the benchmark runs without npm install buildNpmPackage builds node_modules and dist from the lockfile, so smoke, bench, generate and compare run hermetically. nix flake check runs a smoke pass. Two packaging fixes it surfaced: an explicit files allowlist so npm pack stops honoring .gitignore and ships dist/, and node-x12 and x12-parser are runtime dependencies of compare.mjs, not build-time. --- README.md | 19 +++++- flake.lock | 27 ++++++++ flake.nix | 154 ++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 8 +-- package.json | 14 +++-- result | 1 + 6 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 120000 result diff --git a/README.md b/README.md index 13aa597..612e356 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,21 @@ information (PHI) or customer data is used, stored, or required. ## Reproduce -Requires Node >= 23. +With Nix there is nothing to install. The flake builds `node_modules` and +`dist/` from the lockfile, so the apps run without `npm install` and without +network. + +```bash +# from anywhere, no checkout: generate ~5 MB and run it under the cap +nix run github:StanzaAPI/benchmark-harness#smoke + +# full run from a checkout +nix run .#generate -- --transactions 255000 --claims 5 --out data/claims.x12 +nix run .#bench -- data/claims.x12 +nix run .#compare -- --file data/claims.x12 --cap 25 +``` + +`nix run .#help` lists every command. Without Nix, requires Node >= 23: ```bash npm install @@ -36,7 +50,8 @@ node --max-old-space-size=25 bin/run.mjs data/claims.x12 Quick check on ~5 MB instead: ```bash -npm run smoke +nix run .#smoke # with Nix +npm run smoke # without ``` ## Reference results diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..f985f0d --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1789286504, + "narHash": "sha256-eiEK7cKZORNEvX0GeF3RtNEF/JXhgf2RqSp3230q13E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ef34387ddd751e1ab8857adf4676492d32eb24ec", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ef34387ddd751e1ab8857adf4676492d32eb24ec", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..4007477 --- /dev/null +++ b/flake.nix @@ -0,0 +1,154 @@ +{ + description = "Reproducible benchmark for the Stanza zero-regex X12/NDJSON streaming parser"; + + # Pinned to the nixpkgs revision the other Stanza repos use. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/ef34387ddd751e1ab8857adf4676492d32eb24ec"; + + outputs = + { self, nixpkgs }: + let + lib = nixpkgs.lib; + systems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + forAllSystems = f: lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + + helpText = '' + # Stanza streaming parser benchmark + + Reproducible benchmark for the zero-regex X12 streaming parser. All data + is generated locally and is synthetic; no PHI, no customer data. + + ## Try it + + nix run .#smoke generate ~5 MB and run it under a 25 MB cap + nix run .#help this text + + With no checkout: + + nix run github:StanzaAPI/benchmark-harness#smoke + + ## Full runs + + nix run .#generate -- --transactions 255000 --claims 5 --out data/claims.x12 + nix run .#bench -- data/claims.x12 + nix run .#compare -- --file data/claims.x12 --cap 25 + + generate and bench write to the current directory. smoke works in a temp + directory and cleans up after itself. + + ## What it measures + + - wall time to stream one file through iterateX12StreamFrames; + - peak sampled V8 heap and whether the run finishes under a hard + --max-old-space-size=25 cap; + - records per second, where one record is one X12 transaction. + + Read README.md for the reference results and COMPARISON.md for the + head-to-head against node-x12 and x12-parser. + ''; + + tools = + pkgs: + let + node = pkgs.nodejs; + # Hermetic build: node_modules and dist/ come from the lockfile, so + # running the apps needs no npm install and no network. + harness = pkgs.buildNpmPackage { + pname = "stanza-benchmark-harness"; + version = "0.1.0"; + src = ./.; + npmDepsHash = "sha256-XQcxP9ke6wmRMIqCAUqIP3X0YIocUfJctmE+g2OztSk="; + npmBuildScript = "build"; + meta.description = "Reproducible benchmark for the Stanza X12 streaming parser"; + }; + dir = "${harness}/lib/node_modules/stanza-benchmark-harness"; + mkApp = + name: extra: text: + pkgs.writeShellApplication { + inherit name text; + runtimeInputs = [ + node + pkgs.coreutils + ] + ++ extra; + }; + apps = { + help = mkApp "help" [ pkgs.glow ] '' + exec glow -s dark ${pkgs.writeText "benchmark-help.md" helpText} + ''; + + generate = mkApp "generate" [ ] '' + exec node ${dir}/bin/generate.mjs "$@" + ''; + + bench = mkApp "bench" [ ] '' + file="''${1:-}" + if [ -z "$file" ]; then + echo "usage: nix run .#bench -- " >&2 + exit 2 + fi + exec node --max-old-space-size=25 ${dir}/bin/run.mjs "$file" + ''; + + compare = mkApp "compare" [ ] '' + exec node ${dir}/bin/compare.mjs "$@" + ''; + + # The one command a stranger runs: generate a small file in a temp + # directory, run it under the cap, clean up. + smoke = mkApp "smoke" [ ] '' + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + node ${dir}/bin/generate.mjs --transactions 5000 --claims 5 --out "$tmp/smoke.x12" + exec node --max-old-space-size=25 ${dir}/bin/run.mjs "$tmp/smoke.x12" + ''; + }; + in + { + inherit harness apps; + }; + in + { + packages = forAllSystems ( + pkgs: + let + t = tools pkgs; + in + t.apps // { default = t.harness; } + ); + + apps = forAllSystems ( + pkgs: + lib.mapAttrs (_: drv: { + type = "app"; + program = lib.getExe drv; + }) (tools pkgs).apps + ); + + checks = forAllSystems (pkgs: { + smoke = + pkgs.runCommand "benchmark-smoke" + { + nativeBuildInputs = [ pkgs.nodejs ]; + } + '' + dir=${(tools pkgs).harness}/lib/node_modules/stanza-benchmark-harness + node "$dir/bin/generate.mjs" --transactions 2000 --claims 4 --out smoke.x12 + node --max-old-space-size=25 "$dir/bin/run.mjs" smoke.x12 + touch $out + ''; + }); + + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + packages = [ pkgs.nodejs ]; + }; + }); + + formatter = forAllSystems (pkgs: pkgs.nixfmt); + }; +} diff --git a/package-lock.json b/package-lock.json index 1b04806..cbc0bfd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,13 +10,11 @@ "license": "BSL-1.1", "dependencies": { "@types/node": "^22.20.4", + "node-x12": "^1.7.1", "typescript": "^5.9.3", + "x12-parser": "^1.3.0", "zod": "^4.6.5" }, - "devDependencies": { - "node-x12": "^1.7.1", - "x12-parser": "^1.3.0" - }, "engines": { "node": ">=23" } @@ -34,7 +32,6 @@ "version": "1.7.1", "resolved": "https://registry.npmjs.org/node-x12/-/node-x12-1.7.1.tgz", "integrity": "sha512-Iv7W218+RY+aebKunYFxjel5qnwVsfiQpsOv35kkPxV860rA7tpIz15bLzKqQy9mzbqQAT1dSXPD+y4S91EE+A==", - "dev": true, "license": "MIT" }, "node_modules/typescript": { @@ -60,7 +57,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/x12-parser/-/x12-parser-1.3.0.tgz", "integrity": "sha512-KBb+PodQJBtLJ4DkBS0j1GmyRDfwuAnou6DXx0vIoPnqoghABhhtd8u5YIdagHf0TE2OxmoqMOcVXeooqkS6Rw==", - "dev": true, "license": "MIT", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 77c6ad1..61bf176 100644 --- a/package.json +++ b/package.json @@ -17,13 +17,19 @@ "compare": "node bin/compare.mjs --file data/claims.x12", "compare:capped": "node bin/compare.mjs --file data/claims.x12 --cap 25" }, + "files": [ + "bin", + "src", + "dist", + "README.md", + "COMPARISON.md", + "LICENSE" + ], "dependencies": { "@types/node": "^22.20.4", + "node-x12": "^1.7.1", "typescript": "^5.9.3", + "x12-parser": "^1.3.0", "zod": "^4.6.5" - }, - "devDependencies": { - "node-x12": "^1.7.1", - "x12-parser": "^1.3.0" } } diff --git a/result b/result new file mode 120000 index 0000000..ed0e63c --- /dev/null +++ b/result @@ -0,0 +1 @@ +/nix/store/b9yzcga19llpghxyqzsknpsxnc1pg71p-stanza-benchmark-harness-0.1.0 \ No newline at end of file