diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index caeff82..b9e5402 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ permissions: jobs: server: - runs-on: macos-14 + runs-on: macos-15 timeout-minutes: 30 defaults: run: @@ -17,11 +17,11 @@ jobs: env: HF_HUB_DISABLE_TELEMETRY: "1" steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.1.0 with: enable-cache: true - - uses: actions/cache@v4 + - uses: actions/cache@v6 with: path: ~/.cache/huggingface/hub key: models-${{ hashFiles('server/src/system_one_lite/data/*_answer_codes.json') }} @@ -34,11 +34,11 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.1.0 with: enable-cache: true - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: 22 cache: npm @@ -52,3 +52,9 @@ jobs: - name: Type-check JavaScript SDK working-directory: sdks/javascript run: npm run typecheck + - name: Test JavaScript SDK package + working-directory: sdks/javascript + run: npm test + - name: Check JavaScript SDK build output + working-directory: sdks/javascript + run: git diff --exit-code -- dist diff --git a/.gitignore b/.gitignore index dc85acb..ce4e1f6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ htmlcov/ venv/ *.egg-info/ dist/ +!sdks/javascript/dist/ +!sdks/javascript/dist/** build/ # JavaScript diff --git a/README.md b/README.md index 6d1a9f7..e80e381 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # System One Lite -a tiny project turns a normal local LLM into a typed decision engine. It needs no fine-tuning, text generation, or parser. That is enough to build a surprising amount of software. [docs](docs/index.md) +A tiny project that turns a normal local LLM into a typed decision engine. It +needs no fine-tuning, text generation, or parser. [Read the docs](docs/index.md). ## Stop asking language models to write. Start making them decide. @@ -23,10 +24,11 @@ question. No free-form response. No JSON repair loop. No invented option that your code has never heard of. -This is an independent proof of concept for the interface behind -[System One Models](https://typesafe.ai/blog/introducing-system-one-models-and-jev). -It is not a new foundation model. It runs a stock open-weight model locally -with MLX and asks a much more interesting question: +This is an independent proof of concept. + +It uses the System One Models interface. +It is not a new foundation model. It runs a stock open-weight model with MLX. +The project tests simpler AI software where the model can only decide. ## The old stack is absurd @@ -184,18 +186,18 @@ and the reason this safe path uses one model pass per question. ## Extraordinary claims, meet a local eval -There is no benchmark confetti here. The repository has an eval runner so you -can measure the idea on the machine that will run it: +There is no benchmark confetti here. The repository has an eval runner for +JSONL files that use the documented dataset envelope: ```bash cd server -uv run python -m tools.evals --limit 20 +uv run python -m tools.evals --datasets /path/to/jsonl-directory --limit 20 ``` The report shows accuracy by question type. It also rotates Choice options and -checks whether changing their order changes the winner. The repository includes -generated examples, public benchmark samples, and game states with exact or -policy-derived labels under [`datasets/`](datasets/README.md). +checks whether changing their order changes the winner. The public dataset is +the next release step and is not in Git yet. Local files under `datasets/` are +ignored, so they cannot be published by accident. Better yet, add examples from your own traffic. A decision system earns trust on the states it will actually see, not on a launch graphic. @@ -239,8 +241,13 @@ uv run ruff check src tools tests uv run ruff format --check src tools tests uv run pytest -cd ../sdks/javascript +cd ../sdks/python +uv run --with pytest pytest -q + +cd ../javascript +npm ci npm run typecheck +npm test ``` ## Project map diff --git a/sdks/javascript/README.md b/sdks/javascript/README.md index 3639012..0957ef4 100644 --- a/sdks/javascript/README.md +++ b/sdks/javascript/README.md @@ -1,8 +1,14 @@ # JavaScript SDK (local clone) -A local clone of the `system-sdk` API, pointed at the System One -Lite server (`server/src/system_one_lite/api.py`). Source is TypeScript; Node 22.18 or newer -runs it directly without a flag or build step. Calls time out after 30 seconds. +A local clone of the `system-sdk` API, pointed at the System One Lite server +(`server/src/system_one_lite/api.py`). It builds TypeScript to JavaScript for +Node 22.18 or newer. Calls time out after 30 seconds. + +Install it from a System One Lite checkout: + +```bash +npm install /path/to/system-one/sdks/javascript +``` ## Use @@ -41,5 +47,5 @@ adds `score` and `legend`; Noul is a single `noul` number. - `src/client.ts` — `SystemClient` with `systemOne`. - `examples/quickstart.ts` — the docs example plus all three question types. -Install development dependencies with `npm install`, then type-check with -`npm run typecheck`. +Install development dependencies with `npm install`. Run `npm run typecheck` +and `npm test` before publishing or using a changed package. diff --git a/sdks/javascript/dist/client.d.ts b/sdks/javascript/dist/client.d.ts new file mode 100644 index 0000000..06d2ef3 --- /dev/null +++ b/sdks/javascript/dist/client.d.ts @@ -0,0 +1,13 @@ +import type { Content, QuestionBuilder, SystemOneResponse } from "./types.ts"; +export interface SystemClientOptions { + baseUrl?: string; +} +export interface SystemOneRequest> { + state: Content; + questions: QS; +} +export declare class SystemClient { + private readonly baseUrl; + constructor(options?: SystemClientOptions); + systemOne>(request: SystemOneRequest): Promise>; +} diff --git a/sdks/javascript/dist/client.js b/sdks/javascript/dist/client.js new file mode 100644 index 0000000..b88389c --- /dev/null +++ b/sdks/javascript/dist/client.js @@ -0,0 +1,21 @@ +export class SystemClient { + baseUrl; + constructor(options = {}) { + this.baseUrl = (options.baseUrl ?? process.env.SYSTEM_BASE_URL ?? "http://127.0.0.1:8010").replace(/\/+$/, ""); + } + async systemOne(request) { + const response = await fetch(`${this.baseUrl}/evaluate`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + state: request.state, + questions: Object.fromEntries(Object.entries(request.questions).map(([name, builder]) => [name, builder.question])), + }), + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + throw new Error(`systemOne failed: HTTP ${response.status} ${(await response.text()).slice(0, 200)}`); + } + return (await response.json()); + } +} diff --git a/sdks/javascript/dist/index.d.ts b/sdks/javascript/dist/index.d.ts new file mode 100644 index 0000000..c865f4e --- /dev/null +++ b/sdks/javascript/dist/index.d.ts @@ -0,0 +1,4 @@ +export { SystemClient } from "./client.ts"; +export type { SystemOneRequest, SystemClientOptions } from "./client.ts"; +export { choice, noul, score } from "./types.ts"; +export type { Answers, ChoiceAnswer, ChoiceBuilder, ChoiceQuestion, Content, NoulAnswer, NoulBuilder, NoulQuestion, QuestionBuilder, ScoreAnswer, ScoreBuilder, ScoreQuestion, SystemOneResponse, Usage, } from "./types.ts"; diff --git a/sdks/javascript/dist/index.js b/sdks/javascript/dist/index.js new file mode 100644 index 0000000..b2ea496 --- /dev/null +++ b/sdks/javascript/dist/index.js @@ -0,0 +1,2 @@ +export { SystemClient } from "./client.js"; +export { choice, noul, score } from "./types.js"; diff --git a/sdks/javascript/dist/types.d.ts b/sdks/javascript/dist/types.d.ts new file mode 100644 index 0000000..66a4e59 --- /dev/null +++ b/sdks/javascript/dist/types.d.ts @@ -0,0 +1,76 @@ +export type Content = string | Record | unknown[]; +export interface NoulQuestion { + type: "noul"; + instructions: Content; + criteria?: { + true?: string; + false?: string; + }; +} +export interface ChoiceQuestion { + type: "choice"; + instructions: Content; + criteria: Record; +} +export interface ScoreQuestion { + type: "score"; + instructions: Content; + criteria: Content[]; +} +export interface NoulBuilder { + readonly kind: "noul"; + readonly question: NoulQuestion; +} +export interface ChoiceBuilder { + readonly kind: "choice"; + readonly question: ChoiceQuestion; +} +export interface ScoreBuilder { + readonly kind: "score"; + readonly question: ScoreQuestion; +} +export type QuestionBuilder = NoulBuilder | ChoiceBuilder | ScoreBuilder; +export declare function noul(instructions: Content, criteria?: { + true?: string; + false?: string; +}): NoulBuilder; +export declare function choice(instructions: Content, options: { + [key in K]: Content | null; +}): ChoiceBuilder; +export declare function score(instructions: Content, levels: Content[]): ScoreBuilder; +export interface NoulAnswer { + type: "noul"; + noul: number; +} +export interface ChoiceAnswer { + type: "choice"; + choice: K; + probabilities: Record; + confidence: number; +} +export interface ScoreAnswer { + type: "score"; + score: number; + legend: Record; + probabilities: Record; + confidence: number; +} +export type AnswerFor = Q extends { + question: ChoiceQuestion; +} ? ChoiceAnswer : Q extends { + question: NoulQuestion; +} ? NoulAnswer : Q extends { + question: ScoreQuestion; +} ? ScoreAnswer : never; +export type Answers> = { + [K in keyof QS]: AnswerFor; +}; +export interface Usage { + input_tokens: number; + output_tokens: number; +} +export interface SystemOneResponse> { + model: string; + answers: Answers; + usage: Usage; +} diff --git a/sdks/javascript/dist/types.js b/sdks/javascript/dist/types.js new file mode 100644 index 0000000..406d916 --- /dev/null +++ b/sdks/javascript/dist/types.js @@ -0,0 +1,14 @@ +// Question and answer types. Builders carry their option keys so the +// response's answers map is fully inferred from the questions you send. +export function noul(instructions, criteria) { + const question = criteria + ? { type: "noul", instructions, criteria } + : { type: "noul", instructions }; + return { kind: "noul", question }; +} +export function choice(instructions, options) { + return { kind: "choice", question: { type: "choice", instructions, criteria: options } }; +} +export function score(instructions, levels) { + return { kind: "score", question: { type: "score", instructions, criteria: levels } }; +} diff --git a/sdks/javascript/package.json b/sdks/javascript/package.json index 6ab3b69..ee439df 100644 --- a/sdks/javascript/package.json +++ b/sdks/javascript/package.json @@ -4,10 +4,19 @@ "description": "Local JavaScript SDK clone for the System One Lite evaluate endpoint", "license": "MIT", "type": "module", + "files": [ + "dist" + ], "exports": { - ".": "./src/index.ts" + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } }, "scripts": { + "build": "tsc -p tsconfig.build.json", + "prepack": "npm run build", + "test": "npm run build && node --test", "typecheck": "tsc --noEmit" }, "engines": { diff --git a/sdks/javascript/tests/package.test.mjs b/sdks/javascript/tests/package.test.mjs new file mode 100644 index 0000000..6d05e3a --- /dev/null +++ b/sdks/javascript/tests/package.test.mjs @@ -0,0 +1,11 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { choice, noul, score, SystemClient } from "system-sdk"; + +test("the built package exports its public API", () => { + assert.equal(choice("pick", { yes: null }).question.type, "choice"); + assert.equal(noul("check").question.type, "noul"); + assert.equal(score("rate", ["low", "high"]).question.type, "score"); + assert.equal(typeof SystemClient, "function"); +}); diff --git a/sdks/javascript/tsconfig.build.json b/sdks/javascript/tsconfig.build.json new file mode 100644 index 0000000..e514429 --- /dev/null +++ b/sdks/javascript/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "noEmit": false, + "outDir": "dist", + "rewriteRelativeImportExtensions": true, + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +}