Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,21 @@ information (PHI) or customer data is used, stored, or required.

## Reproduce

Requires Node >= 23.
With Nix there is nothing to install. The flake builds `node_modules` and
`dist/` from the lockfile, so the apps run without `npm install` and without
network.

```bash
# from anywhere, no checkout: generate ~5 MB and run it under the cap
nix run github:StanzaAPI/benchmark-harness#smoke

# full run from a checkout
nix run .#generate -- --transactions 255000 --claims 5 --out data/claims.x12
nix run .#bench -- data/claims.x12
nix run .#compare -- --file data/claims.x12 --cap 25
```

`nix run .#help` lists every command. Without Nix, requires Node >= 23:

```bash
npm install
Expand All @@ -36,7 +50,8 @@ node --max-old-space-size=25 bin/run.mjs data/claims.x12
Quick check on ~5 MB instead:

```bash
npm run smoke
nix run .#smoke # with Nix
npm run smoke # without
```

## Reference results
Expand Down
27 changes: 27 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

154 changes: 154 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
{
description = "Reproducible benchmark for the Stanza zero-regex X12/NDJSON streaming parser";

# Pinned to the nixpkgs revision the other Stanza repos use.
inputs.nixpkgs.url = "github:NixOS/nixpkgs/ef34387ddd751e1ab8857adf4676492d32eb24ec";

outputs =
{ self, nixpkgs }:
let
lib = nixpkgs.lib;
systems = [
"x86_64-linux"
"aarch64-linux"
"x86_64-darwin"
"aarch64-darwin"
];
forAllSystems = f: lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});

helpText = ''
# Stanza streaming parser benchmark

Reproducible benchmark for the zero-regex X12 streaming parser. All data
is generated locally and is synthetic; no PHI, no customer data.

## Try it

nix run .#smoke generate ~5 MB and run it under a 25 MB cap
nix run .#help this text

With no checkout:

nix run github:StanzaAPI/benchmark-harness#smoke

## Full runs

nix run .#generate -- --transactions 255000 --claims 5 --out data/claims.x12
nix run .#bench -- data/claims.x12
nix run .#compare -- --file data/claims.x12 --cap 25

generate and bench write to the current directory. smoke works in a temp
directory and cleans up after itself.

## What it measures

- wall time to stream one file through iterateX12StreamFrames;
- peak sampled V8 heap and whether the run finishes under a hard
--max-old-space-size=25 cap;
- records per second, where one record is one X12 transaction.

Read README.md for the reference results and COMPARISON.md for the
head-to-head against node-x12 and x12-parser.
'';

tools =
pkgs:
let
node = pkgs.nodejs;
# Hermetic build: node_modules and dist/ come from the lockfile, so
# running the apps needs no npm install and no network.
harness = pkgs.buildNpmPackage {
pname = "stanza-benchmark-harness";
version = "0.1.0";
src = ./.;
npmDepsHash = "sha256-XQcxP9ke6wmRMIqCAUqIP3X0YIocUfJctmE+g2OztSk=";
npmBuildScript = "build";
meta.description = "Reproducible benchmark for the Stanza X12 streaming parser";
};
dir = "${harness}/lib/node_modules/stanza-benchmark-harness";
mkApp =
name: extra: text:
pkgs.writeShellApplication {
inherit name text;
runtimeInputs = [
node
pkgs.coreutils
]
++ extra;
};
apps = {
help = mkApp "help" [ pkgs.glow ] ''
exec glow -s dark ${pkgs.writeText "benchmark-help.md" helpText}
'';

generate = mkApp "generate" [ ] ''
exec node ${dir}/bin/generate.mjs "$@"
'';

bench = mkApp "bench" [ ] ''
file="''${1:-}"
if [ -z "$file" ]; then
echo "usage: nix run .#bench -- <file.x12>" >&2
exit 2
fi
exec node --max-old-space-size=25 ${dir}/bin/run.mjs "$file"
'';

compare = mkApp "compare" [ ] ''
exec node ${dir}/bin/compare.mjs "$@"
'';

# The one command a stranger runs: generate a small file in a temp
# directory, run it under the cap, clean up.
smoke = mkApp "smoke" [ ] ''
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
node ${dir}/bin/generate.mjs --transactions 5000 --claims 5 --out "$tmp/smoke.x12"
exec node --max-old-space-size=25 ${dir}/bin/run.mjs "$tmp/smoke.x12"
'';
};
in
{
inherit harness apps;
};
in
{
packages = forAllSystems (
pkgs:
let
t = tools pkgs;
in
t.apps // { default = t.harness; }
);

apps = forAllSystems (
pkgs:
lib.mapAttrs (_: drv: {
type = "app";
program = lib.getExe drv;
}) (tools pkgs).apps
);

checks = forAllSystems (pkgs: {
smoke =
pkgs.runCommand "benchmark-smoke"
{
nativeBuildInputs = [ pkgs.nodejs ];
}
''
dir=${(tools pkgs).harness}/lib/node_modules/stanza-benchmark-harness
node "$dir/bin/generate.mjs" --transactions 2000 --claims 4 --out smoke.x12
node --max-old-space-size=25 "$dir/bin/run.mjs" smoke.x12
touch $out
'';
});

devShells = forAllSystems (pkgs: {
default = pkgs.mkShell {
packages = [ pkgs.nodejs ];
};
});

formatter = forAllSystems (pkgs: pkgs.nixfmt);
};
}
8 changes: 2 additions & 6 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 10 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,19 @@
"compare": "node bin/compare.mjs --file data/claims.x12",
"compare:capped": "node bin/compare.mjs --file data/claims.x12 --cap 25"
},
"files": [
"bin",
"src",
"dist",
"README.md",
"COMPARISON.md",
"LICENSE"
],
"dependencies": {
"@types/node": "^22.20.4",
"node-x12": "^1.7.1",
"typescript": "^5.9.3",
"x12-parser": "^1.3.0",
"zod": "^4.6.5"
},
"devDependencies": {
"node-x12": "^1.7.1",
"x12-parser": "^1.3.0"
}
}
1 change: 1 addition & 0 deletions result
Loading