From 4f24f97b6e231864fd454bf8ac6c6a9fb7485bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Wed, 28 Oct 2026 19:21:07 +0100 Subject: [PATCH 1/2] build(cli): Only build native binaries for the current platform Cross-platform builds with build-native never produced a working binary, and nothing uses them; every CI job builds on its own platform. Two things break them, both silently at build time: - pkg compiles bytecode on the host, and V8 on the target rejects it ("V8 rejected the bytecode cache", seen building for win-x64 on Linux and running under wine); - native addons are packaged as yarn install compiled them. pkg's prebuild-install step, which is meant to fetch an addon for the target, looks for prebuild-install under @yao-pkg/pkg/node_modules, where it is not installed in this workspace. Even when reachable, its node-abi 3.57.0 maps Node 22 to ABI 124 instead of 127, so the download 404s. Failures are logged at debug level only, and pkg falls back to the host addon ("Bad EXE format" under wine). So build-native is now a Node script that only builds for the current platform. OS and ARCH stay as optional arguments, so the CI calls are unchanged, but now they only assert the platform and a mismatch is an error. The pkg target is taken from the package's pkg.targets, which becomes the single place the Node version is declared. It always passes --no-native-build, so it no longer depends on is_native, which never matched on Windows ("windows" vs pkg's "win"). That made no practical difference, given the above: Windows also ended up with the addon yarn install built. Root `yarn build-native` used to try every target; it now builds each package for the current platform. The scripts call the file through `node` so they do not rely on shebang handling on Windows; scanner's bin/build-native stays a symlink to this one. Assisted-by: Claude:claude-opus-5-5 --- packages/cli/bin/build-native | 129 ++++++++++++--------- packages/cli/package.json | 2 +- packages/cli/tests/binary/helpers/setup.ts | 4 +- packages/scanner/package.json | 2 +- 4 files changed, 78 insertions(+), 59 deletions(-) diff --git a/packages/cli/bin/build-native b/packages/cli/bin/build-native index d8cbfa372f..24055eac74 100755 --- a/packages/cli/bin/build-native +++ b/packages/cli/bin/build-native @@ -1,57 +1,76 @@ -#!/bin/bash -set -e - -# Optionally pass a desired OS as the first argument, followed by an optional architecture. -# e.g. ./build-native macos arm64 -# -# Otherwise every target will be built. -OS="${1}" -ARCH="${2:-x64}" -ARGS="--config package.json --compress Brotli" -PKG_NAME=$(node -p "require('./package.json').name.split('/').pop()") - -# check if OS is set and OS/ARCH is the same as the current system. -is_native() { - [ -z "${OS}" ] && return 1 - # Check if the current system matches the specified OS and ARCH. - case "$(uname -s)" in - Linux*) current_os="linux" ;; - Darwin*) current_os="macos" ;; - CYGWIN*) current_os="windows" ;; - MINGW*) current_os="windows" ;; - *) current_os="unknown" ;; - esac - current_arch="$(uname -m)" - case "${current_arch}" in - x86_64) current_arch="x64" ;; - arm64) current_arch="arm64" ;; - aarch64) current_arch="arm64" ;; - *) current_arch="unknown" ;; - esac - [ "${OS}" = "${current_os}" ] && [ "${ARCH}" = "${current_arch}" ] +#!/usr/bin/env node + +// Builds a native binary of the package in the current directory for the +// current platform, with pkg. Shared by packages/cli and packages/scanner +// (whose bin/build-native is a symlink to this file). +// +// build-native [OS [ARCH]] +// +// OS and ARCH are pkg's names (linux, macos, win; x64, arm64). They are +// optional and only asserted against the current platform: cross-platform +// builds are not supported. Two things break them. pkg compiles bytecode on +// the host, and V8 on the target rejects it. And native addons +// (better-sqlite3) are packaged as yarn install compiled them, because pkg's +// own prebuild-install step cannot find prebuild-install in this workspace and +// would request the wrong ABI anyway (its node-abi maps Node 22 to 124). Both +// failures are silent at build time and only show when the binary runs. +// +// The pkg target comes from the package's pkg.targets, which is therefore the +// single place the supported platforms and the Node version are declared. + +const { spawnSync } = require('child_process'); +const { resolve } = require('path'); + +const hostOs = { linux: 'linux', darwin: 'macos', win32: 'win' }[process.platform]; +const hostArch = process.arch; + +const [os = hostOs, arch = hostArch] = process.argv.slice(2); +if (os !== hostOs || arch !== hostArch) { + console.error( + `build-native: cannot build for ${os}-${arch} on ${hostOs}-${hostArch}; ` + + 'only builds for the current platform are supported.' + ); + process.exit(1); } -# Do not rebuild native binary addons if the architecture and system match. -# These should be built by yarn install beforehand. -# Having pkg try to rebuild them risks downloading pre-built versions that do not work -# correctly on the target machine. -if is_native; then - echo "Building native binary for current system ${OS} ${ARCH} - skipping native build step." - ARGS="${ARGS} --no-native-build" -fi - -# NOTE: The Node version target (e.g. node22) MUST match the Node version configured -# in GHA workflows and actions (e.g. setup-node/action.yml). If they mismatch, native Node addons -# (like better-sqlite3) compiled during GHA host execution will have ABI version mismatches -# (NODE_MODULE_VERSION mismatches) at runtime in the packaged executable. -# -# Note that Node 24+ officially dropped support for macOS < 13.5 due to newer libc++ requirements. -# Keeping the target at Node 22 ensures backward compatibility with older macOS versions. -if [ -n "${OS}" ]; then - echo "Building native binaries for ${OS} ${ARCH}..." - ARGS="${ARGS} -o release/${PKG_NAME}-${OS}-${ARCH} -t node22-${OS}-${ARCH}" -else - echo "Building native binaries for each target..." -fi - -yarn pkg ${ARGS} built/cli.js +const packageJson = require(resolve('package.json')); +const pkgName = packageJson.name.split('/').pop(); +const target = packageJson.pkg.targets.find((t) => t.endsWith(`-${os}-${arch}`)); +if (!target) { + console.error(`build-native: no pkg target for ${os}-${arch} in package.json`); + process.exit(1); +} + +// NOTE: The Node version of the targets (e.g. node22) MUST match the Node +// version configured in GHA workflows and actions (e.g. setup-node/action.yml), +// because native addons compiled by yarn install on the runner are packaged +// as-is (see --no-native-build below). +// +// Note that Node 24+ officially dropped support for macOS < 13.5 due to newer +// libc++ requirements. Keeping the target at Node 22 ensures backward +// compatibility with older macOS versions. + +console.log(`Building native binary for ${target}...`); + +// --no-native-build: package the addons yarn install built rather than let pkg +// try to fetch prebuilt ones. CI deliberately compiles them against old glibc +// (in the RHEL 8 container) and with MACOSX_DEPLOYMENT_TARGET. +const args = [ + 'pkg', + '--config', + 'package.json', + '--compress', + 'Brotli', + '--no-native-build', + '-t', + target, + '-o', + `release/${pkgName}-${os}-${arch}`, + 'built/cli.js', +]; +const { status, error } = spawnSync('yarn', args, { + stdio: 'inherit', + shell: process.platform === 'win32', +}); +if (error) throw error; +process.exit(status ?? 1); diff --git a/packages/cli/package.json b/packages/cli/package.json index 7679e7541d..f5dbb36768 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -27,7 +27,7 @@ "build:ui:css": "tailwindcss -i src/html/query-ui/index.css -o built/html/query-ui/tailwind.css --config src/html/query-ui/tailwind.config.js --minify", "build:doc": "node ../../scripts/sync-docs.mjs --source ../../docs --target built/docs", "start": "ts-node src/cli.ts", - "build-native": "yarn build && ./bin/build-native" + "build-native": "yarn build && node bin/build-native" }, "lint-staged": { "*.{js,md,json,ts}": [ diff --git a/packages/cli/tests/binary/helpers/setup.ts b/packages/cli/tests/binary/helpers/setup.ts index 71ad2cc233..4cd7a34c02 100644 --- a/packages/cli/tests/binary/helpers/setup.ts +++ b/packages/cli/tests/binary/helpers/setup.ts @@ -39,8 +39,8 @@ export default async function performSetup() { try { await run( new CommandStruct( - './bin/build-native', - [os, arch], + 'node', + ['bin/build-native', os, arch], packageRoot, {} ) diff --git a/packages/scanner/package.json b/packages/scanner/package.json index 0ffbe85fda..622840943c 100644 --- a/packages/scanner/package.json +++ b/packages/scanner/package.json @@ -11,7 +11,7 @@ "types": "built/index.d.ts", "scripts": { "build": "node bin/preBuild.js && tsc -p tsconfig.build.json && yarn schema && yarn doc", - "build-native": "yarn build && ./bin/build-native", + "build-native": "yarn build && node bin/build-native", "start": "ts-node src/cli.ts", "doc": "ts-node ./bin/verify-rules-doc.ts && ts-node ./bin/verify-labels-doc.ts && ts-node ./bin/front-matter.ts", "schema": "ts-node ./bin/schema.ts", From 15d0627dde284e1cbb90d0f593419831295e11e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Rzepecki?= Date: Wed, 28 Oct 2026 19:21:37 +0100 Subject: [PATCH 2/2] build(cli): Check native addon ABI before building the binary build-native packages native addons (better_sqlite3.node) exactly as yarn install compiled them. If that ran on a different Node major than the pkg target (node22), the build still succeeds, after several minutes, and the binary then dies at runtime with ERR_DLOPEN_FAILED (NODE_MODULE_VERSION mismatch). build-native now checks first. It takes the addons from the .node entries in pkg.assets and fails if the current Node major differs from the target's, or if any addon does not load under process.dlopen. The error names the `yarn rebuild` to run. Packages without native assets, like scanner, skip the check. `yarn build-native` runs it via --check-only before `yarn build`, so it fails in under a second instead of after the TypeScript build. Yarn passes "$@" through without also appending the arguments. The binary test setup calls build-native directly and gets the check as part of the build. Assisted-by: Claude:claude-opus-5-5 --- packages/cli/bin/build-native | 56 +++++++++++++++++++++++++++++++++-- packages/cli/package.json | 2 +- packages/scanner/package.json | 2 +- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/cli/bin/build-native b/packages/cli/bin/build-native index 24055eac74..3675b849c2 100755 --- a/packages/cli/bin/build-native +++ b/packages/cli/bin/build-native @@ -4,7 +4,7 @@ // current platform, with pkg. Shared by packages/cli and packages/scanner // (whose bin/build-native is a symlink to this file). // -// build-native [OS [ARCH]] +// build-native [--check-only] [OS [ARCH]] // // OS and ARCH are pkg's names (linux, macos, win; x64, arm64). They are // optional and only asserted against the current platform: cross-platform @@ -15,16 +15,23 @@ // would request the wrong ABI anyway (its node-abi maps Node 22 to 124). Both // failures are silent at build time and only show when the binary runs. // +// Before building, checks that the native addons will load in the binary (see +// checkNativeAbi below). With --check-only it stops there, which lets +// `yarn build-native` fail fast before spending time on `yarn build`. +// // The pkg target comes from the package's pkg.targets, which is therefore the // single place the supported platforms and the Node version are declared. const { spawnSync } = require('child_process'); +const { existsSync } = require('fs'); const { resolve } = require('path'); const hostOs = { linux: 'linux', darwin: 'macos', win32: 'win' }[process.platform]; const hostArch = process.arch; -const [os = hostOs, arch = hostArch] = process.argv.slice(2); +const argv = process.argv.slice(2); +const checkOnly = argv.includes('--check-only'); +const [os = hostOs, arch = hostArch] = argv.filter((arg) => arg !== '--check-only'); if (os !== hostOs || arch !== hostArch) { console.error( `build-native: cannot build for ${os}-${arch} on ${hostOs}-${hostArch}; ` + @@ -50,6 +57,51 @@ if (!target) { // libc++ requirements. Keeping the target at Node 22 ensures backward // compatibility with older macOS versions. +// Native addons are packaged as yarn install compiled them, so they only load +// in the binary if that was on the target's Node major. Otherwise the build +// succeeds and the binary dies at runtime with ERR_DLOPEN_FAILED +// (NODE_MODULE_VERSION mismatch). +function checkNativeAbi() { + const addons = packageJson.pkg.assets + .filter((asset) => asset.endsWith('.node')) + .map((asset) => resolve(asset)) + .filter((path) => existsSync(path)); + if (addons.length === 0) return; + + const targetMajor = target.match(/^node(\d+)-/)[1]; + const packages = [ + ...new Set( + addons.flatMap( + (path) => path.match(/node_modules[\\/]((?:@[^\\/]+[\\/])?[^\\/]+)/)?.[1] ?? [] + ) + ), + ]; + const fail = (reason) => { + console.error(`build-native: ${reason} + +The binary targets Node ${targetMajor}, and native addons are packaged as +yarn install compiled them, so they must be built for that Node major. + +Switch to Node ${targetMajor} (e.g. \`nvm use ${targetMajor}\`), run +\`yarn rebuild ${packages.join(' ')}\`, then retry.`); + process.exit(1); + }; + + if (process.versions.node.split('.')[0] !== targetMajor) + fail(`running on Node ${process.versions.node}.`); + for (const addon of addons) { + try { + process.dlopen({ exports: {} }, addon); + } catch (err) { + fail(`${addon} does not load on Node ${process.versions.node}: + ${String(err.message).replace(/\n/g, '\n ')}`); + } + } +} + +checkNativeAbi(); +if (checkOnly) process.exit(0); + console.log(`Building native binary for ${target}...`); // --no-native-build: package the addons yarn install built rather than let pkg diff --git a/packages/cli/package.json b/packages/cli/package.json index f5dbb36768..d6568c7700 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -27,7 +27,7 @@ "build:ui:css": "tailwindcss -i src/html/query-ui/index.css -o built/html/query-ui/tailwind.css --config src/html/query-ui/tailwind.config.js --minify", "build:doc": "node ../../scripts/sync-docs.mjs --source ../../docs --target built/docs", "start": "ts-node src/cli.ts", - "build-native": "yarn build && node bin/build-native" + "build-native": "node bin/build-native --check-only \"$@\" && yarn build && node bin/build-native \"$@\"" }, "lint-staged": { "*.{js,md,json,ts}": [ diff --git a/packages/scanner/package.json b/packages/scanner/package.json index 622840943c..3d80efb704 100644 --- a/packages/scanner/package.json +++ b/packages/scanner/package.json @@ -11,7 +11,7 @@ "types": "built/index.d.ts", "scripts": { "build": "node bin/preBuild.js && tsc -p tsconfig.build.json && yarn schema && yarn doc", - "build-native": "yarn build && node bin/build-native", + "build-native": "node bin/build-native --check-only \"$@\" && yarn build && node bin/build-native \"$@\"", "start": "ts-node src/cli.ts", "doc": "ts-node ./bin/verify-rules-doc.ts && ts-node ./bin/verify-labels-doc.ts && ts-node ./bin/front-matter.ts", "schema": "ts-node ./bin/schema.ts",