diff --git a/AGENTS.md b/AGENTS.md index 4bebedf..e3ee32d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,7 @@ # AGENTS.md -An extension of the -[Deno Standard Library](https://github.com/denoland/deno_std), published on JSR -under the `@stdext` scope. +An extension of the [Deno Standard Library](https://github.com/denoland/std), +published on JSR under the `@stdext` scope. ## Repository layout diff --git a/README.md b/README.md index a14e45c..2c65387 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # Deno Standard Library Extended [![JSR @stdext](https://jsr.io/badges/@stdext)](https://jsr.io/@stdext) -[![codecov](https://codecov.io/gh/halvardssm/deno_stdext/graph/badge.svg?token=T1JEMGF8VW)](https://codecov.io/gh/halvardssm/deno_stdext) -[![ci](https://github.com/halvardssm/deno_stdext/actions/workflows/ci.yml/badge.svg)](https://github.com/halvardssm/deno_stdext/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/halvardssm/std-js/graph/badge.svg?token=T1JEMGF8VW)](https://codecov.io/gh/halvardssm/std-js) +[![ci](https://github.com/halvardssm/std-js/actions/workflows/ci.yml/badge.svg)](https://github.com/halvardssm/std-js/actions/workflows/ci.yml) -An extension of the -[Deno Standard Library](https://github.com/denoland/deno_std). +An extension of the [Deno Standard Library](https://github.com/denoland/std). Multiple languages such as Rust, Go and PHP offer a standard library, which allows the reduction of third-party libraries. This project is used to extend @@ -46,6 +45,8 @@ console.log(dump(buffer)); - [assert](https://jsr.io/@stdext/assert): The assert package, contains validators and assertions +- [cli](https://jsr.io/@stdext/cli): The cli package contains a command + framework and helpers for building command line applications - [collections](https://jsr.io/@stdext/collections): The collections package contains commonly used utilities and structures - [crypto](https://jsr.io/@stdext/crypto): The crypto package contains utility @@ -74,13 +75,15 @@ console.log(dump(buffer)); with fully inferred types - [xml](https://jsr.io/@stdext/xml): The xml package provides XML parsing, serialization, and XSD validation, backed by a WebAssembly implementation +- [zip](https://jsr.io/@stdext/zip): The zip package contains helpers for + creating and extracting zip archives ## Platform support The packages are primarily developed and tested on macOS and Linux. Windows support is not guaranteed, and packages may not work as expected on Windows. If you encounter an issue on Windows, please -[report it](https://github.com/halvardssm/deno_stdext/issues). +[report it](https://github.com/halvardssm/std-js/issues). ## Versioning @@ -125,7 +128,7 @@ more details. ## Deprecation Policy We follow the -[Deno STD Deprecation Policy](https://github.com/denoland/deno_std?tab=readme-ov-file#deprecation-policy). +[Deno STD Deprecation Policy](https://github.com/denoland/std?tab=readme-ov-file#deprecation-policy). When functionalities are adopted either by JavaScript language APIs, new Web Standard APIs or Deno STD, we mark the functions as deprecated, and will remove @@ -147,9 +150,9 @@ the license allows for it. To bump versions and publish the packages to JSR, do the following: 1. Trigger the manual GitHub action - [version bump](https://github.com/halvardssm/stdext/actions/workflows/version_bump.yml). + [version bump](https://github.com/halvardssm/std-js/actions/workflows/version_bump.yml). 2. Check the created PR, verify the changes, and merge. -3. Create a [new release](https://github.com/halvardssm/stdext/releases/new), +3. Create a [new release](https://github.com/halvardssm/std-js/releases/new), with the tag name `release-[date]` (e.g. `release-2024.12.29`) and auto generate release notes. diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 0000000..7b95ef0 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,97 @@ +# @stdext/cli + +Extends [@std/cli](https://jsr.io/@std/cli) + +The cli package contains helpers for building command line applications. + +## Commands + +`defineCommand` and `runCommand` build a CLI from plain objects. The flags and +arguments given to `run` are typed from the definition. + +```ts ignore +import { defineCommand, runCommand } from "@stdext/cli"; + +const serve = defineCommand({ + name: "serve", + version: "2.1.0", + description: "Start the server", + options: { + port: { type: "string", alias: "p", default: "8000", env: "MYTOOL_PORT" }, + config: { type: "string", required: true, env: "MYTOOL_CONFIG" }, + dryRun: { type: "boolean" }, + }, + args: [{ name: "dir", required: true }], + run({ flags, args, rest, stdout }) { + // flags.port: string, flags.config: string, flags.dryRun: boolean + stdout(`Serving ${args.dir} on ${flags.port}`); + }, +}); + +const cli = defineCommand({ + name: "mytool", + version: "1.0.0", + description: "An example tool", + commands: [serve], +}); + +Deno.exit(await runCommand(cli, Deno.args)); +``` + +``` +$ mytool serve ./public --config app.toml -p 3000 +$ MYTOOL_PORT=3000 mytool serve ./public --config app.toml +$ mytool serve --help +$ mytool serve --version +mytool 1.0.0 +mytool serve 2.1.0 +``` + +- **Options** are `string` or `boolean`, and are passed as `--name`, where a + camelCase name is written `--camel-case`. An `alias` is a single character + passed as `-p`. +- **Value order:** a flag takes its value from the arguments, then its `env` + variable, then its `default`. An empty variable counts as not set. Boolean + variables accept `1`, `true`, `yes` and `on`, and `0`, `false`, `no` and + `off`. +- **Required** string options fail when none of the three provides a value. +- **Arguments** are positional, and can be `required`, or `variadic` for the + last one. Everything after `--` is passed as `rest`, unparsed. +- **Subcommands** are chosen by the first arguments, so the options of a command + come after its name. +- **Help and version:** `--help` / `-h` show generated help for the command. + `--version` / `-V` print the version of the command, and of its parent + commands, one per line, e.g. `mytool 1.0.0` and `mytool serve 2.1.0`. Any + command can have a `version`, and commands without one are left out. Replace + the help with the `help` option of `runCommand`. +- **Help when empty:** set `helpOnEmpty: true` on a command to show its help, + and succeed, when it is called without any arguments or flags. This takes the + place of running the command, or failing on a missing required argument, or on + `Missing command` for a command with subcommands. +- **Suggestions:** an unknown command or `--option` is reported with the closest + known one, e.g. `Unknown command 'serv'. Did you mean 'serve'?`, when it is + within two edits. Configure it with the `suggest` option of `runCommand`: + `false` turns it off, and `{ maxDistance: 3 }` changes the distance. +- **Errors:** a usage error, such as an unknown flag or a missing required + option, is written to stderr and exits with code `2`, or the `usageExitCode` + option of `runCommand`. A command can throw `UsageError` as well. Other errors + are not caught. +- **Testing:** `runCommand` returns the exit code, and takes `stdout`, `stderr` + and `env` options, so commands can be tested without subprocesses. + +## parseArgs + +`parseArgs` is a drop in replacement for the one in `@std/cli`, with a +`required` option: a list of flags that throws when they are not present. The +flags in `required`, and in `default`, are not `undefined` in the type of the +result. + +```ts +import { parseArgs } from "@stdext/cli/parse-args"; + +const args = parseArgs(["--name", "foo"], { + string: ["name"], + required: ["name"], +}); +const name: string = args.name; +``` diff --git a/cli/_types.ts b/cli/_types.ts new file mode 100644 index 0000000..6ebc8df --- /dev/null +++ b/cli/_types.ts @@ -0,0 +1,227 @@ +// Copyright 2018-2026 the Deno authors. MIT license. +// Types copied from `@std/cli/parse-args`, where they are internal and not +// exported. They are needed to infer the result of `parseArgs` from its options +// the same way as `@std/cli` does. + +/** Combines recursively all intersection types and returns a new single type. + * @internal + */ +type Id = TRecord extends Record + ? TRecord extends infer InferredRecord + ? { [Key in keyof InferredRecord]: Id } + : never + : TRecord; + +/** Converts a union type `A | B | C` into an intersection type `A & B & C`. + * @internal + */ +type UnionToIntersection = + (TValue extends unknown ? (args: TValue) => unknown : never) extends + (args: infer R) => unknown ? R extends Record ? R : never + : never; + +/** @internal */ +export type BooleanType = boolean | string | undefined; +/** @internal */ +export type StringType = string | undefined; +/** @internal */ +type ArgType = StringType | BooleanType; + +/** @internal */ +export type Collectable = string | undefined; +/** @internal */ +export type Negatable = string | undefined; + +type UseTypes< + TBooleans extends BooleanType, + TStrings extends StringType, + TCollectable extends Collectable, +> = undefined extends ( + & (false extends TBooleans ? undefined : TBooleans) + & TCollectable + & TStrings +) ? false + : true; + +/** + * Creates a record with all available flags with the corresponding type and + * default type. + * @internal + */ +export type Values< + TBooleans extends BooleanType, + TStrings extends StringType, + TCollectable extends Collectable, + TNegatable extends Negatable, + TDefault extends Record | undefined, + TAliases extends Aliases | undefined, +> = UseTypes extends true ? + & Record + & AddAliases< + SpreadDefaults< + & CollectValues + & RecursiveRequired> + & CollectUnknownValues< + TBooleans, + TStrings, + TCollectable, + TNegatable + >, + DedotRecord + >, + TAliases + > + // deno-lint-ignore no-explicit-any + : Record; + +/** @internal */ +export type Aliases = + Partial< + Record, TAliasNames | ReadonlyArray> + >; + +type AddAliases< + TArgs, + TAliases extends Aliases | undefined, +> = { + [TArgName in keyof TArgs as AliasNames]: TArgs[TArgName]; +}; + +type AliasNames< + TArgName, + TAliases extends Aliases | undefined, +> = TArgName extends keyof TAliases + ? string extends TAliases[TArgName] ? TArgName + : TAliases[TArgName] extends string ? TArgName | TAliases[TArgName] + : TAliases[TArgName] extends Array + ? TArgName | TAliases[TArgName][number] + : TArgName + : TArgName; + +/** + * Spreads all default values of Record `TDefaults` into Record `TArgs` + * and makes default values required. + * + * **Example:** + * `SpreadValues<{ foo?: boolean, bar?: number }, { foo: number }>` + * + * **Result:** `{ foo: boolean | number, bar?: number }` + */ +type SpreadDefaults = TDefaults extends undefined ? TArgs + : TArgs extends Record ? + & Omit + & { + [Default in keyof TDefaults]: Default extends keyof TArgs + ? (TArgs[Default] & TDefaults[Default] | TDefaults[Default]) extends + Record + ? NonNullable> + : TDefaults[Default] | NonNullable + : unknown; + } + : never; + +/** + * Defines the Record for the `default` option to add + * auto-suggestion support for IDE's. + * @internal + */ +type Defaults = Id< + UnionToIntersection< + & Record + // Dedotted auto suggestions: { foo: { bar: unknown } } + & MapTypes + & MapTypes + // Flat auto suggestions: { "foo.bar": unknown } + & MapDefaults + & MapDefaults + > +>; + +type MapDefaults = Partial< + Record +>; + +type RecursiveRequired = TRecord extends Record ? { + [Key in keyof TRecord]-?: RecursiveRequired; + } + : TRecord; + +/** Same as `MapTypes` but also supports collectable options. */ +type CollectValues< + TArgNames extends ArgType, + TType, + TCollectable extends Collectable, + TNegatable extends Negatable = undefined, +> = UnionToIntersection< + Extract extends string ? + & (Exclude extends never ? Record + : MapTypes, TType, TNegatable>) + & (Extract extends never ? Record + : RecursiveRequired< + MapTypes, Array, TNegatable> + >) + : MapTypes +>; + +/** Same as `Record` but also supports dotted and negatable options. */ +type MapTypes< + TArgNames extends ArgType, + TType, + TNegatable extends Negatable = undefined, +> = undefined extends TArgNames ? Record + : TArgNames extends `${infer Name}.${infer Rest}` ? { + [Key in Name]?: MapTypes< + Rest, + TType, + TNegatable extends `${Name}.${infer Negate}` ? Negate : undefined + >; + } + : TArgNames extends string ? Partial< + Record + > + : Record; + +type CollectUnknownValues< + TBooleans extends BooleanType, + TStrings extends StringType, + TCollectable extends Collectable, + TNegatable extends Negatable, +> = UnionToIntersection< + TCollectable extends TBooleans & TStrings ? Record + : DedotRecord< + // Unknown collectable & non-negatable args. + & Record< + Exclude< + Extract, string>, + Extract + >, + Array + > + // Unknown collectable & negatable args. + & Record< + Exclude< + Extract, string>, + Extract + >, + Array | false + > + > +>; + +/** Converts `{ "foo.bar.baz": unknown }` into `{ foo: { bar: { baz: unknown } } }`. */ +type DedotRecord = Record extends TRecord ? TRecord + : TRecord extends Record ? UnionToIntersection< + ValueOf< + { + [Key in keyof TRecord]: Key extends string ? Dedot + : never; + } + > + > + : TRecord; + +type Dedot = TKey extends + `${infer Name}.${infer Rest}` ? { [Key in Name]: Dedot } + : { [Key in TKey]: TValue }; + +type ValueOf = TValue[keyof TValue]; diff --git a/cli/command.test.ts b/cli/command.test.ts new file mode 100644 index 0000000..c6ef60e --- /dev/null +++ b/cli/command.test.ts @@ -0,0 +1,854 @@ +import { assertEquals, assertThrows } from "@std/assert"; +import { + defineCommand, + runCommand, + type RunOptions, + UsageError, +} from "./command.ts"; + +Deno.test("defineCommand returns the definition", () => { + const definition = { name: "tool", description: "d" }; + assertEquals(defineCommand(definition), definition); +}); + +Deno.test("defineCommand infers flags and arguments", () => { + defineCommand({ + name: "tool", + options: { + plain: { type: "string" }, + required: { type: "string", required: true }, + optional: { type: "string", required: false }, + withDefault: { type: "string", default: "x" }, + withEnv: { type: "string", env: "X" }, + switch: { type: "boolean" }, + }, + args: [ + { name: "first", required: true }, + { name: "second" }, + { name: "others", variadic: true }, + ], + run({ flags, args }) { + const plain: string | undefined = flags.plain; + const required: string = flags.required; + const optional: string | undefined = flags.optional; + const withDefault: string = flags.withDefault; + const withEnv: string | undefined = flags.withEnv; + const toggle: boolean = flags.switch; + const first: string = args.first; + const second: string | undefined = args.second; + const others: string[] = args.others; + // @ts-expect-error `plain` can be undefined + const definitelyPlain: string = flags.plain; + // @ts-expect-error `second` is optional + const definitelySecond: string = args.second; + // @ts-expect-error the flag does not exist + flags.nothing; + // @ts-expect-error the argument does not exist + args.nothing; + return [ + plain, + required, + optional, + withDefault, + withEnv, + toggle, + first, + second, + others, + definitelyPlain, + definitelySecond, + ].length; + }, + }); +}); + +Deno.test("defineCommand validates the definition", async (t) => { + const invalid: Array<[string, () => unknown, string]> = [ + ["empty name", () => defineCommand({ name: "" }), "Invalid command name"], + [ + "name with space", + () => defineCommand({ name: "a b" }), + "Invalid command name", + ], + [ + "name like a flag", + () => defineCommand({ name: "--a" }), + "Invalid command name", + ], + [ + "reserved option", + () => + defineCommand({ name: "a", options: { help: { type: "boolean" } } }), + "reserved", + ], + [ + "reserved alias", + () => + defineCommand({ + name: "a", + options: { x: { type: "boolean", alias: "h" } }, + }), + "reserved", + ], + [ + "reserved version", + () => + defineCommand({ + name: "a", + options: { x: { type: "boolean", alias: "V" } }, + }), + "reserved", + ], + [ + "duplicate alias", + () => + defineCommand({ + name: "a", + options: { + x: { type: "boolean", alias: "a" }, + y: { type: "boolean", alias: "a" }, + }, + }), + "more than once", + ], + [ + "duplicate flag", + () => + defineCommand({ + name: "a", + options: { + dryRun: { type: "boolean" }, + "dry-run": { type: "boolean" }, + }, + }), + "more than once", + ], + [ + "long alias", + () => + defineCommand({ + name: "a", + options: { x: { type: "boolean", alias: "ab" } }, + }), + "not one character", + ], + [ + "subcommands and arguments", + () => + defineCommand({ + name: "a", + args: [{ name: "x" }], + commands: [defineCommand({ name: "b" })], + }), + "both subcommands and arguments", + ], + [ + "duplicate subcommand", + () => + defineCommand({ + name: "a", + commands: [ + defineCommand({ name: "b" }), + defineCommand({ name: "b" }), + ], + }), + "twice", + ], + [ + "variadic not last", + () => + defineCommand({ + name: "a", + args: [{ name: "x", variadic: true }, { name: "y" }], + }), + "not last", + ], + [ + "required after optional", + () => + defineCommand({ + name: "a", + args: [{ name: "x" }, { name: "y", required: true }], + }), + "follows an optional", + ], + ]; + for (const [name, fn, message] of invalid) { + await t.step(name, () => { + assertThrows(fn, TypeError, message); + }); + } +}); + +type Result = { code: number; out: string[]; err: string[] }; + +async function run( + command: Parameters[0], + args: string[], + env: Record = {}, + options: RunOptions = {}, +): Promise { + const out: string[] = []; + const err: string[] = []; + const code = await runCommand(command, args, { + stdout: (line) => void out.push(line), + stderr: (line) => void err.push(line), + env: (name) => env[name], + ...options, + }); + return { code, out, err }; +} + +const serve = defineCommand({ + name: "serve", + version: "2.0.0", + description: "Start the server", + options: { + port: { type: "string", alias: "p", default: "8000", env: "PORT" }, + config: { type: "string", required: true, env: "CONFIG" }, + dryRun: { type: "boolean", alias: "d", env: "DRY_RUN" }, + name: { type: "string" }, + }, + args: [{ name: "dir", required: true }, { name: "more", variadic: true }], + run({ flags, args, rest, stdout }) { + stdout(JSON.stringify({ flags, args, rest })); + }, +}); + +const cli = defineCommand({ + name: "tool", + version: "1.2.3", + description: "A tool", + commands: [ + serve, + defineCommand({ + name: "group", + commands: [defineCommand({ + name: "leaf", + run: ({ stdout }) => stdout("leaf"), + })], + }), + defineCommand({ name: "code", run: () => 7 }), + defineCommand({ name: "async", run: async () => {}, options: {} }), + defineCommand({ name: "empty" }), + ], +}); + +const parsed = (result: Result) => JSON.parse(result.out[0]!); + +Deno.test("runCommand runs commands and subcommands", async (t) => { + await t.step("runs the root command", async () => { + const result = await run( + defineCommand({ name: "x", run: ({ stdout }) => stdout("hi") }), + [], + ); + assertEquals(result, { code: 0, out: ["hi"], err: [] }); + }); + + await t.step("runs a subcommand", async () => { + assertEquals((await run(cli, ["group", "leaf"])).out, ["leaf"]); + }); + + await t.step("uses the returned exit code", async () => { + assertEquals((await run(cli, ["code"])).code, 7); + assertEquals((await run(cli, ["async"])).code, 0); + }); + + await t.step("passes the context", async () => { + const result = await run(cli, [ + "serve", + "dir", + "--config=c", + "-d", + "a", + "b", + "--", + "--x", + "y", + ]); + assertEquals(result.code, 0); + assertEquals(parsed(result), { + flags: { port: "8000", config: "c", dryRun: true }, + args: { dir: "dir", more: ["a", "b"] }, + rest: ["--x", "y"], + }); + }); + + await t.step("uses console by default", async () => { + const original = console.log; + const lines: unknown[] = []; + console.log = (line: unknown) => lines.push(line); + try { + assertEquals(await runCommand(cli, ["--version"]), 0); + } finally { + console.log = original; + } + assertEquals(lines, ["tool 1.2.3"]); + }); +}); + +Deno.test("runCommand flags", async (t) => { + const base = ["serve", "d", "--config", "c"]; + + await t.step("passed, then env, then default", async () => { + const env = { PORT: "1", CONFIG: "e" }; + assertEquals( + parsed(await run(cli, [...base, "--port", "2"], env)).flags.port, + "2", + ); + assertEquals(parsed(await run(cli, base, env)).flags.port, "1"); + assertEquals(parsed(await run(cli, base)).flags.port, "8000"); + }); + + await t.step("aliases and kebab case", async () => { + assertEquals(parsed(await run(cli, [...base, "-p", "3"])).flags.port, "3"); + assertEquals( + parsed(await run(cli, [...base, "--dry-run"])).flags.dryRun, + true, + ); + }); + + await t.step("optional string is undefined", async () => { + assertEquals("name" in parsed(await run(cli, base)).flags, false); + assertEquals( + parsed(await run(cli, [...base, "--name", "n"])).flags.name, + "n", + ); + }); + + await t.step( + "a passed value wins over the env, even when empty", + async () => { + const result = await run(cli, [...base, "--port="], { PORT: "1" }); + assertEquals(parsed(result).flags.port, ""); + }, + ); + + await t.step("an empty variable is not set", async () => { + assertEquals(parsed(await run(cli, base, { PORT: "" })).flags.port, "8000"); + assertEquals( + (await run(cli, ["serve", "d"], { CONFIG: "" })).code, + 2, + ); + }); + + await t.step("required is satisfied by the env", async () => { + const result = await run(cli, ["serve", "d"], { CONFIG: "fromEnv" }); + assertEquals(parsed(result).flags.config, "fromEnv"); + }); + + await t.step("boolean env values", async () => { + for (const value of ["1", "true", "YES", "on"]) { + assertEquals( + parsed(await run(cli, base, { DRY_RUN: value })).flags.dryRun, + true, + ); + } + for (const value of ["0", "False", "no", "off"]) { + assertEquals( + parsed(await run(cli, base, { DRY_RUN: value })).flags.dryRun, + false, + ); + } + }); + + await t.step("a passed boolean wins over the env", async () => { + const result = await run(cli, [...base, "--dry-run=false"], { + DRY_RUN: "1", + }); + assertEquals(parsed(result).flags.dryRun, false); + }); + + await t.step("a boolean defaults to false, or its default", async () => { + assertEquals(parsed(await run(cli, base)).flags.dryRun, false); + const withDefault = defineCommand({ + name: "x", + options: { on: { type: "boolean", default: true } }, + run: ({ flags, stdout }) => stdout(String(flags.on)), + }); + assertEquals((await run(withDefault, [])).out, ["true"]); + assertEquals((await run(withDefault, ["--on=false"])).out, ["false"]); + }); +}); + +Deno.test("runCommand usage errors", async (t) => { + const expectUsage = async ( + args: string[], + message: string, + env: Record = {}, + hint = "Run 'tool serve --help' for usage.", + ) => { + const result = await run(cli, args, env); + assertEquals(result.code, 2); + assertEquals(result.out, []); + assertEquals(result.err, [`Error: ${message}`, hint]); + }; + + await t.step( + "unknown command", + () => + expectUsage( + ["unrelated"], + "Unknown command 'unrelated'", + {}, + "Run 'tool --help' for usage.", + ), + ); + await t.step( + "unknown nested command", + () => + expectUsage( + ["group", "unrelated"], + "Unknown command 'unrelated'", + {}, + "Run 'tool group --help' for usage.", + ), + ); + await t.step( + "missing command", + () => + expectUsage( + ["group"], + "Missing command", + {}, + "Run 'tool group --help' for usage.", + ), + ); + await t.step( + "command without run", + () => + expectUsage( + ["empty"], + "Command 'empty' cannot be run", + {}, + "Run 'tool empty --help' for usage.", + ), + ); + await t.step( + "missing required option", + () => + expectUsage( + ["serve", "d"], + "Missing required option --config (or environment variable CONFIG)", + ), + ); + await t.step("missing required option without env", async () => { + const command = defineCommand({ + name: "x", + options: { token: { type: "string", required: true } }, + run: () => {}, + }); + const result = await run(command, []); + assertEquals(result.err[0], "Error: Missing required option --token"); + }); + await t.step( + "missing argument", + () => + expectUsage( + ["serve", "--config", "c"], + "Missing required argument ", + ), + ); + await t.step("missing variadic argument", async () => { + const command = defineCommand({ + name: "x", + args: [{ name: "files", required: true, variadic: true }], + run: () => {}, + }); + assertEquals( + (await run(command, [])).err[0], + "Error: Missing required argument ", + ); + assertEquals((await run(command, ["a"])).code, 0); + }); + await t.step( + "unknown option", + () => + expectUsage( + ["serve", "d", "--config", "c", "--zzzzzz=1", "-z"], + "Unknown options: --zzzzzz, -z", + ), + ); + await t.step( + "one unknown option", + () => + expectUsage( + ["serve", "d", "--config", "c", "--zzzzzz"], + "Unknown option: --zzzzzz", + ), + ); + await t.step("unexpected argument", async () => { + const command = defineCommand({ + name: "x", + args: [{ name: "one" }], + run: () => {}, + }); + const result = await run(command, ["a", "b"]); + assertEquals(result.err[0], "Error: Unexpected argument 'b'"); + assertEquals( + (await run(defineCommand({ name: "x", run: () => {} }), ["a"])).code, + 2, + ); + }); + await t.step("invalid boolean env", () => + expectUsage( + ["serve", "d", "--config", "c"], + "Environment variable DRY_RUN is 'maybe', expected one of 1, true, yes, on, 0, false, no, off", + { DRY_RUN: "maybe" }, + )); + await t.step("the exit code can be changed", async () => { + const result = await run(cli, ["unrelated"], {}, { usageExitCode: 1 }); + assertEquals(result.code, 1); + assertEquals(result.err[0], "Error: Unknown command 'unrelated'"); + assertEquals((await run(cli, ["unrelated"])).code, 2); + }); + await t.step("a usage error thrown by the command", async () => { + const command = defineCommand({ + name: "x", + run: () => { + throw new UsageError("bad input"); + }, + }); + const result = await run(command, []); + assertEquals(result.code, 2); + assertEquals(result.err[0], "Error: bad input"); + }); + await t.step("other errors are not caught", async () => { + const command = defineCommand({ + name: "x", + run: () => { + throw new Error("boom"); + }, + }); + let message = ""; + await run(command, []).catch((e) => message = e.message); + assertEquals(message, "boom"); + }); +}); + +Deno.test("runCommand suggestions", async (t) => { + await t.step("a command", async () => { + const result = await run(cli, ["serv"]); + assertEquals( + result.err[0], + "Error: Unknown command 'serv'. Did you mean 'serve'?", + ); + assertEquals(result.code, 2); + }); + + await t.step("is not case sensitive", async () => { + assertEquals( + (await run(cli, ["SERVE2"])).err[0], + "Error: Unknown command 'SERVE2'. Did you mean 'serve'?", + ); + }); + + await t.step("no suggestion when nothing is near", async () => { + assertEquals( + (await run(cli, ["unrelated"])).err[0], + "Error: Unknown command 'unrelated'", + ); + }); + + await t.step("an option", async () => { + const result = await run(cli, [ + "serve", + "d", + "--config", + "c", + "--prot", + "1", + ]); + assertEquals( + result.err[0], + "Error: Unknown option: --prot (did you mean --port?)", + ); + }); + + await t.step("a kebab case option, and several unknown", async () => { + const result = await run(cli, [ + "serve", + "d", + "--config", + "c", + "--dryrun", + "--zzzzzz", + "-x", + ]); + assertEquals( + result.err[0], + "Error: Unknown options: --dryrun (did you mean --dry-run?), --zzzzzz, -x", + ); + }); + + await t.step("--help is suggested too", async () => { + const result = await run(cli, ["serve", "--hepl"]); + assertEquals( + result.err[0], + "Error: Unknown option: --hepl (did you mean --help?)", + ); + }); + + await t.step("a command without options", async () => { + const result = await run(defineCommand({ name: "x", run: () => {} }), [ + "--nope", + ]); + assertEquals(result.err[0], "Error: Unknown option: --nope"); + }); + + await t.step("can be turned off", async () => { + assertEquals( + (await run(cli, ["serv"], {}, { suggest: false })).err[0], + "Error: Unknown command 'serv'", + ); + const result = await run( + cli, + ["serve", "d", "--config", "c", "--prot"], + {}, + { suggest: false }, + ); + assertEquals(result.err[0], "Error: Unknown option: --prot"); + }); + + await t.step("the distance can be changed", async () => { + // `serv` is one edit from `serve`. + assertEquals( + (await run(cli, ["serv"], {}, { suggest: { maxDistance: 0 } })).err[0], + "Error: Unknown command 'serv'", + ); + // `srv` is two edits from `serve`. + assertEquals( + (await run(cli, ["srv"], {}, { suggest: { maxDistance: 1 } })).err[0], + "Error: Unknown command 'srv'", + ); + assertEquals( + (await run(cli, ["srv"], {}, { suggest: { maxDistance: 2 } })).err[0], + "Error: Unknown command 'srv'. Did you mean 'serve'?", + ); + assertEquals( + (await run(cli, ["sr"], {}, { suggest: { maxDistance: 3 } })).err[0], + "Error: Unknown command 'sr'. Did you mean 'serve'?", + ); + assertEquals( + (await run(cli, ["serv"], {}, { suggest: true })).err[0], + "Error: Unknown command 'serv'. Did you mean 'serve'?", + ); + }); +}); + +Deno.test("runCommand help and version", async (t) => { + await t.step("--help and -h", async () => { + for ( + const args of [["--help"], ["-h"], ["serve", "--help"], ["group", "-h"]] + ) { + const result = await run(cli, args); + assertEquals(result.code, 0); + assertEquals(result.err, []); + assertEquals(result.out.length, 1); + } + }); + + await t.step("help is for the resolved command", async () => { + const result = await run(cli, ["serve", "--help"]); + assertEquals(result.out[0]!.includes("Usage: tool serve"), true); + // Required flags and arguments do not need to be given. + assertEquals(result.code, 0); + }); + + await t.step("a flag after -- is not help", async () => { + const command = defineCommand({ + name: "x", + run: ({ rest, stdout }) => stdout(rest.join(",")), + }); + assertEquals((await run(command, ["--", "--help"])).out, ["--help"]); + }); + + await t.step("a custom help renderer", async () => { + const result = await run(cli, ["serve", "-h"], {}, { + help: (path) => path.map((c) => c.name).join("/"), + }); + assertEquals(result.out, ["tool/serve"]); + }); + + await t.step("--version and -V", async () => { + assertEquals((await run(cli, ["--version"])).out, ["tool 1.2.3"]); + assertEquals((await run(cli, ["-V"])).out, ["tool 1.2.3"]); + const noVersion = defineCommand({ name: "x", run: () => {} }); + assertEquals((await run(noVersion, ["--version"])).code, 2); + }); + + await t.step("a subcommand version includes its parents", async () => { + const result = await run(cli, ["serve", "--version"]); + assertEquals(result.code, 0); + assertEquals(result.out, ["tool 1.2.3", "tool serve 2.0.0"]); + assertEquals((await run(cli, ["serve", "-V"])).out, result.out); + }); + + await t.step("the output is written with only the line", async () => { + const calls: unknown[][] = []; + await runCommand(cli, ["serve", "-V"], { + stdout: (...args: unknown[]) => void calls.push(args), + }); + assertEquals(calls, [["tool 1.2.3"], ["tool serve 2.0.0"]]); + }); + + await t.step("a subcommand without a version shows its parents", async () => { + assertEquals((await run(cli, ["group", "leaf", "-V"])).out, ["tool 1.2.3"]); + }); + + await t.step("only the versions that are set are shown", async () => { + const leaf = defineCommand({ + name: "leaf", + version: "0.1.0", + run: () => {}, + }); + const middle = defineCommand({ name: "mid", commands: [leaf] }); + const root = defineCommand({ name: "root", commands: [middle] }); + assertEquals((await run(root, ["mid", "leaf", "--version"])).out, [ + "root mid leaf 0.1.0", + ]); + // No command in the path has a version. + assertEquals((await run(root, ["mid", "--version"])).code, 2); + }); + + await t.step("help shows the version of the command", async () => { + assertEquals( + (await run(cli, ["--help"])).out[0]!.startsWith("tool 1.2.3 - A tool"), + true, + ); + const sub = (await run(cli, ["serve", "--help"])).out[0]!; + assertEquals(sub.startsWith("tool serve 2.0.0 - Start the server"), true); + assertEquals(sub.includes("--version"), true); + // A command without a version in its path has no version flag. + const noVersion = defineCommand({ name: "x", run: () => {} }); + assertEquals( + (await run(noVersion, ["--help"])).out[0]!.includes("--version"), + false, + ); + }); +}); + +Deno.test("runCommand helpOnEmpty", async (t) => { + const ran: string[] = []; + const tool = defineCommand({ + name: "tool", + helpOnEmpty: true, + options: { name: { type: "string", required: true } }, + args: [{ name: "file", required: true }], + run: ({ stdout }) => { + ran.push("tool"); + stdout("ran"); + }, + }); + + await t.step("shows the help without arguments", async () => { + const result = await run(tool, []); + assertEquals(result.code, 0); + assertEquals(result.err, []); + assertEquals(result.out.length, 1); + assertEquals(result.out[0]!.startsWith("Usage: tool"), true); + assertEquals(ran, []); + }); + + await t.step("uses the help renderer", async () => { + const result = await run(tool, [], {}, { help: () => "custom" }); + assertEquals(result.out, ["custom"]); + }); + + await t.step("runs, and checks, when something is provided", async () => { + assertEquals((await run(tool, ["--name", "n", "f"])).out, ["ran"]); + assertEquals((await run(tool, ["f"])).code, 2); + assertEquals((await run(tool, ["--name", "n"])).code, 2); + assertEquals((await run(tool, ["--"])).code, 2); + }); + + await t.step("applies to the command that is called", async () => { + const group = defineCommand({ + name: "group", + helpOnEmpty: true, + commands: [ + defineCommand({ + name: "leaf", + args: [{ name: "file", required: true }], + run: () => {}, + }), + tool, + ], + }); + const root = defineCommand({ name: "root", commands: [group] }); + + const help = await run(root, ["group"]); + assertEquals([help.code, help.out[0]!.startsWith("Usage: root group")], [ + 0, + true, + ]); + // Without the option, a command fails on what is missing. + assertEquals((await run(root, [])).err[0], "Error: Missing command"); + assertEquals( + (await run(root, ["group", "leaf"])).err[0], + "Error: Missing required argument ", + ); + // `tool` has the option, and shows its own help. + assertEquals( + (await run(root, ["group", "tool"])).out[0]!.startsWith( + "Usage: root group tool", + ), + true, + ); + }); + + await t.step("is off by default", async () => { + const plain = defineCommand({ + name: "plain", + args: [{ name: "file", required: true }], + run: () => {}, + }); + assertEquals((await run(plain, [])).code, 2); + const noArguments = defineCommand({ + name: "x", + run: ({ stdout }) => stdout("ran"), + }); + assertEquals((await run(noArguments, [])).out, ["ran"]); + }); +}); + +Deno.test("runCommand reads the environment", async (t) => { + const command = defineCommand({ + name: "x", + options: { v: { type: "string", env: "STDEXT_CLI_TEST_VAR" } }, + run: ({ flags, stdout }) => stdout(String(flags.v)), + }); + + await t.step("from the process", async () => { + const out: string[] = []; + Deno.env.set("STDEXT_CLI_TEST_VAR", "from-process"); + try { + await runCommand(command, [], { stdout: (line) => out.push(line) }); + } finally { + Deno.env.delete("STDEXT_CLI_TEST_VAR"); + } + assertEquals(out, ["from-process"]); + }); + + await t.step("a variable that is not readable is not set", async () => { + const script = ` + import { defineCommand, runCommand } from "${ + import.meta.resolve("./command.ts") + }"; + const command = defineCommand({ + name: "x", + options: { v: { type: "string", env: "STDEXT_CLI_TEST_VAR" } }, + run: ({ flags, stdout }) => stdout(String(flags.v)), + }); + Deno.exit(await runCommand(command, [])); + `; + const output = await new Deno.Command(Deno.execPath(), { + args: ["eval", "--no-prompt", "--deny-env", script], + cwd: new URL("../", import.meta.url), + stdout: "piped", + stderr: "piped", + }).output(); + assertEquals(new TextDecoder().decode(output.stderr), ""); + assertEquals(new TextDecoder().decode(output.stdout), "undefined\n"); + }); +}); diff --git a/cli/command.ts b/cli/command.ts new file mode 100644 index 0000000..8198ce3 --- /dev/null +++ b/cli/command.ts @@ -0,0 +1,650 @@ +import { parseArgs } from "@std/cli/parse-args"; +import { closestString } from "@std/text/closest-string"; +import { levenshteinDistance } from "@std/text/levenshtein-distance"; +import { toKebabCase } from "@std/text/to-kebab-case"; + +/** Shared properties of all options */ +type OptionBase = { + /** Description shown in the help */ + description?: string; + /** Single character alias, used as `-p` */ + alias?: string; + /** + * Name of an environment variable to read the value from when the flag is not + * passed. An empty variable counts as not set. + */ + env?: string; +}; + +/** An option that takes a value */ +export type StringOption = OptionBase & { + type: "string"; + /** Value used when the flag, and the environment variable, is not set */ + default?: string; + /** Fail when the flag, the environment variable and default are all unset */ + required?: boolean; +}; + +/** + * An option that is a switch, which is `false` unless it is passed, or set by + * the environment variable. The variable accepts `1`, `true`, `yes` and `on`, + * and `0`, `false`, `no` and `off`. + */ +export type BooleanOption = OptionBase & { + type: "boolean"; + /** Value used when the flag, and the environment variable, is not set */ + default?: boolean; +}; + +/** An option, which is passed as `--name`. A camelCase name is `--camel-case`. */ +export type Option = StringOption | BooleanOption; + +/** The options of a command, by name */ +export type Options = Record; + +/** A positional argument */ +export type Argument = { + /** Name of the argument, which is also its key in the context */ + name: string; + /** Description shown in the help */ + description?: string; + /** Fail when the argument is missing */ + required?: boolean; + /** Collect all remaining arguments, which is only possible for the last one */ + variadic?: boolean; +}; + +type FlagValue = TOption extends { type: "boolean" } ? boolean + : TOption extends { default: string } | { required: true } ? string + : string | undefined; + +/** The type of the parsed flags of a command */ +export type Flags = { + [TName in keyof TOptions]: FlagValue; +}; + +type ArgumentValue = TArgument extends { variadic: true } ? string[] + : TArgument extends { required: true } ? string + : string | undefined; + +/** The type of the parsed positional arguments of a command */ +export type Arguments> = { + [TArg in TArguments[number] as TArg["name"]]: ArgumentValue; +}; + +/** What a command is run with */ +export type Context< + TOptions extends Options = Options, + TArguments extends ReadonlyArray = ReadonlyArray, +> = { + /** The parsed flags */ + flags: Flags; + /** The parsed positional arguments, by name */ + args: Arguments; + /** The arguments after `--`, which are not parsed */ + rest: string[]; + /** Writes a line to the output, instead of the console, so it can be tested */ + stdout: (text: string) => void; + /** Writes a line to the error output */ + stderr: (text: string) => void; +}; + +/** The definition of a command, see {@linkcode defineCommand} */ +export type CommandDefinition< + TOptions extends Options, + TArguments extends ReadonlyArray, +> = { + /** Name of the command, which is how it is called */ + name: string; + /** Description shown in the help */ + description?: string; + /** + * Version, which enables `--version` for the command and its subcommands. + * It prints the versions of the parent commands too, when they have one. + */ + version?: string; + /** The options of the command */ + options?: TOptions; + /** The positional arguments of the command */ + args?: TArguments; + /** + * Shows the help, and succeeds, when the command is called without any + * arguments or flags, instead of running it or failing on what is missing. + * A command with subcommands fails with `Missing command` otherwise. + */ + helpOnEmpty?: boolean; + /** The subcommands, which are chosen by the first argument */ + commands?: ReadonlyArray; + /** + * Runs the command. It can return an exit code, which is `0` by default. + * A command with subcommands does not need one. + */ + run?: ( + context: Context, + ) => void | number | Promise; +}; + +/** A command, made with {@linkcode defineCommand} */ +export type Command< + TOptions extends Options = Options, + TArguments extends ReadonlyArray = ReadonlyArray, +> = Readonly>; + +/** A command with any options and arguments, e.g. to list subcommands */ +// deno-lint-ignore no-explicit-any +export type AnyCommand = Command; + +/** + * Defines a command, and checks the definition. + * + * The flags and arguments in the context of `run` are typed from the options + * and arguments: a string flag is a `string` when it is `required` or has a + * `default`, and `string | undefined` otherwise. + * + * @param definition the {@linkcode CommandDefinition} of the command + * @returns the command, to run with `runCommand` + * @throws {TypeError} when the definition is invalid, e.g. when an alias is + * used twice, or an optional argument is followed by a required one + * + * @example + * ```ts + * import { defineCommand } from "@stdext/cli/command"; + * import { assertEquals } from "@std/assert"; + * + * const serve = defineCommand({ + * name: "serve", + * options: { + * port: { type: "string", alias: "p", default: "8000", env: "PORT" }, + * verbose: { type: "boolean" }, + * }, + * args: [{ name: "dir", required: true }], + * run({ flags, args }) { + * const port: string = flags.port; + * const verbose: boolean = flags.verbose; + * const dir: string = args.dir; + * console.log(port, verbose, dir); + * }, + * }); + * + * assertEquals(serve.name, "serve"); + * ``` + */ +export function defineCommand< + const TOptions extends Options = Record, + const TArguments extends ReadonlyArray = readonly [], +>( + definition: CommandDefinition, +): Command { + const { name, args = [], commands = [] } = definition; + + if (!name || /\s/.test(name) || name.startsWith("-")) { + throw new TypeError(`Invalid command name '${name}'`); + } + + const taken = new Set([HELP.long, HELP.short, VERSION.long, VERSION.short]); + const claim = (flag: string, option: string) => { + if (taken.has(flag)) { + throw new TypeError( + `Command '${name}' uses '${flag}' for '${option}' more than once, or it is reserved`, + ); + } + taken.add(flag); + }; + for (const [option, { alias }] of optionsOf(definition)) { + claim(toKebabCase(option), option); + if (alias !== undefined) { + if (alias.length !== 1) { + throw new TypeError( + `Alias '${alias}' of '${option}' is not one character`, + ); + } + claim(alias, option); + } + } + + if (commands.length > 0 && args.length > 0) { + throw new TypeError(`Command '${name}' has both subcommands and arguments`); + } + const names = new Set(); + for (const command of commands) { + if (names.has(command.name)) { + throw new TypeError(`Command '${name}' has '${command.name}' twice`); + } + names.add(command.name); + } + + args.forEach((argument, index) => { + if (argument.variadic && index !== args.length - 1) { + throw new TypeError(`Variadic argument '${argument.name}' is not last`); + } + if (argument.required && args.slice(0, index).some((a) => !a.required)) { + throw new TypeError( + `Required argument '${argument.name}' follows an optional argument`, + ); + } + }); + + return definition; +} + +// Lists the versions of the commands from the root to a command, as lines like +// `tool serve 1.0.0`. Commands without a version are left out. +function versions(path: ReadonlyArray): string[] { + return path.flatMap((command, index) => + command.version + ? [ + `${ + path.slice(0, index + 1).map((c) => c.name).join(" ") + } ${command.version}`, + ] + : [] + ); +} + +/** + * Renders the help of a command as plain text + * + * @param path the commands from the root to the command to show help for + * @returns the help text + * + * @example + * ```ts + * import { defineCommand, renderHelp } from "@stdext/cli/command"; + * import { assert } from "@std/assert"; + * + * const cli = defineCommand({ name: "tool", description: "Does things" }); + * + * assert(renderHelp([cli]).includes("Usage: tool")); + * ``` + */ +export function renderHelp(path: ReadonlyArray): string { + const command = path[path.length - 1]!; + const names = path.map((c) => c.name).join(" "); + const sections: string[] = []; + + const title = [ + command.version ? `${names} ${command.version}` : "", + command.description ?? "", + ].filter(Boolean).join(" - "); + if (title) sections.push(title); + + const args = argsOf(command); + const commands = command.commands ?? []; + const usage = [ + names, + commands.length > 0 ? "" : "", + "[options]", + ...args.map(({ name, required, variadic }) => { + const label = variadic ? `${name}...` : name; + return required ? `<${label}>` : `[${label}]`; + }), + ].filter(Boolean).join(" "); + sections.push(`Usage: ${usage}`); + + if (commands.length > 0) { + sections.push( + `Commands:\n${table(commands.map((c) => [c.name, c.description ?? ""]))}`, + ); + } + + if (args.length > 0) { + sections.push( + `Arguments:\n${table(args.map((a) => [a.name, a.description ?? ""]))}`, + ); + } + + const rows = optionsOf(command).map(([name, option]): [string, string] => { + const flag = `${option.alias ? `-${option.alias}, ` : " "}--${ + toKebabCase(name) + }${option.type === "string" ? " " : ""}`; + const notes = [ + option.description, + option.type === "string" && option.required ? "(required)" : undefined, + option.default !== undefined ? `(default: ${option.default})` : undefined, + option.env ? `[env: ${option.env}]` : undefined, + ].filter(Boolean).join(" "); + return [flag, notes]; + }); + rows.push([`-${HELP.short}, --${HELP.long}`, "Show this help"]); + if (versions(path).length > 0) { + rows.push([`-${VERSION.short}, --${VERSION.long}`, "Show the version"]); + } + sections.push(`Options:\n${table(rows)}`); + + return sections.join("\n\n"); +} + +/** + * An error caused by how the command was called, e.g. an unknown flag. It is + * reported without a stack trace, and exits with code `2` by default. + */ +export class UsageError extends Error { + /** Creates the error with a message for the user of the command */ + constructor(message: string) { + super(message); + this.name = "UsageError"; + } +} + +/** Options for {@linkcode runCommand}, which default to the console and env */ +export type RunOptions = { + /** Writes a line to the output */ + stdout?: (text: string) => void; + /** Writes a line to the error output */ + stderr?: (text: string) => void; + /** Reads an environment variable. A variable that is not readable is unset. */ + env?: (name: string) => string | undefined; + /** + * Suggests the closest command or option when one is not known, e.g. + * `Did you mean 'serve'?`. `false` turns it off, and `maxDistance` is the + * most edits (insertions, deletions and substitutions) that a suggestion can + * be away from what was typed, which is `2` by default. + */ + suggest?: boolean | { maxDistance?: number }; + /** The exit code for a usage error, which is `2` by default */ + usageExitCode?: number; + /** Renders the help of a command, given the commands from the root to it */ + help?: (path: ReadonlyArray) => string; +}; + +const HELP = { long: "help", short: "h" }; +const VERSION = { long: "version", short: "V" }; +const DEFAULT_MAX_DISTANCE = 2; +const NOT_PASSED = Symbol("not passed"); +const TRUE = ["1", "true", "yes", "on"]; +const FALSE = ["0", "false", "no", "off"]; + +/** + * Runs a command with the given arguments + * + * The first arguments choose the subcommand, so the options of a command come + * after its name. A flag takes its value from, in order, the arguments, its + * environment variable and its default. `--help` and `-h` show the help of the + * command, and `--version` and `-V` its version, and the versions of its parents. + * + * An unknown command or option is reported with the closest known one, when it + * is near enough, see the `suggest` option. + * + * Usage errors, like an unknown flag, are written to stderr with the exit code + * `2`, or the `usageExitCode` option. Other errors thrown by the command are not + * caught. + * + * @param command the root command + * @param args the arguments, usually `Deno.args` + * @param options the {@linkcode RunOptions} + * @returns the exit code, which is `0` unless the command returns another + * + * @example + * ```ts + * import { defineCommand, runCommand } from "@stdext/cli/command"; + * import { assertEquals } from "@std/assert"; + * + * const cli = defineCommand({ + * name: "greet", + * options: { name: { type: "string", env: "NAME", default: "world" } }, + * run({ flags, stdout }) { + * stdout(`Hello, ${flags.name}!`); + * }, + * }); + * + * const lines: string[] = []; + * const code = await runCommand(cli, ["--name", "Deno"], { + * stdout: (line) => lines.push(line), + * }); + * + * assertEquals([code, lines], [0, ["Hello, Deno!"]]); + * ``` + */ +export async function runCommand( + command: AnyCommand, + args: readonly string[], + options: RunOptions = {}, +): Promise { + const stdout = options.stdout ?? console.log; + const stderr = options.stderr ?? console.error; + const env = options.env ?? readEnv; + const help = options.help ?? renderHelp; + const usageExitCode = options.usageExitCode ?? 2; + const maxDistance = options.suggest === false + ? undefined + : typeof options.suggest === "object" + ? options.suggest.maxDistance ?? DEFAULT_MAX_DISTANCE + : DEFAULT_MAX_DISTANCE; + const suggest = (input: string, candidates: string[]) => + maxDistance === undefined + ? undefined + : closest(input, candidates, maxDistance); + + const path: AnyCommand[] = [command]; + try { + const tokens = resolveCommand(path, args, suggest); + const current = path[path.length - 1]!; + + const end = tokens.indexOf("--"); + const flagTokens = end < 0 ? tokens : tokens.slice(0, end); + if (hasFlag(flagTokens, HELP)) { + stdout(help(path)); + return 0; + } + if (current.helpOnEmpty && tokens.length === 0) { + stdout(help(path)); + return 0; + } + const versionLines = versions(path); + if (versionLines.length > 0 && hasFlag(flagTokens, VERSION)) { + for (const line of versionLines) stdout(line); + return 0; + } + + if (!current.run) { + throw new UsageError( + current.commands?.length + ? "Missing command" + : `Command '${current.name}' cannot be run`, + ); + } + + const { flags, positionals, rest } = parse(current, tokens, env, suggest); + const exitCode = await current.run({ + flags: flags as never, + args: assign(current, positionals) as never, + rest, + stdout, + stderr, + }); + return exitCode ?? 0; + } catch (error) { + if (!(error instanceof UsageError)) throw error; + stderr(`Error: ${error.message}`); + stderr(`Run '${path.map((c) => c.name).join(" ")} --help' for usage.`); + return usageExitCode; + } +} + +// Follows the leading arguments down the subcommands, adding them to the path +// of commands from the root, and returns the arguments that are left. The path +// is given, so that it is complete for the error when a command is not known. +function resolveCommand( + path: AnyCommand[], + args: readonly string[], + suggest: (input: string, candidates: string[]) => string | undefined, +): string[] { + let tokens = [...args]; + for (;;) { + const current = path[path.length - 1]!; + const name = tokens[0]; + if (!current.commands?.length || name === undefined) break; + if (name.startsWith("-")) break; + const next = current.commands.find((c) => c.name === name); + if (!next) { + const match = suggest(name, current.commands.map((c) => c.name)); + throw new UsageError( + `Unknown command '${name}'${match ? `. Did you mean '${match}'?` : ""}`, + ); + } + path.push(next); + tokens = tokens.slice(1); + } + return tokens; +} + +function hasFlag( + tokens: string[], + { long, short }: { long: string; short: string }, +): boolean { + return tokens.some((token) => token === `--${long}` || token === `-${short}`); +} + +function optionsOf(command: AnyCommand): Array<[string, Option]> { + return Object.entries((command.options ?? {}) as Options); +} + +function argsOf(command: AnyCommand): ReadonlyArray { + return (command.args ?? []) as ReadonlyArray; +} + +function parse( + command: AnyCommand, + tokens: string[], + env: (name: string) => string | undefined, + suggest: (input: string, candidates: string[]) => string | undefined, +) { + const options = optionsOf(command); + const byType = (type: Option["type"]) => + options.filter(([, o]) => o.type === type).map(([name]) => + toKebabCase(name) + ); + + const positionals: string[] = []; + const unknownFlags: string[] = []; + const parsed = parseArgs(tokens, { + string: byType("string"), + boolean: byType("boolean"), + alias: Object.fromEntries( + options.filter(([, o]) => o.alias).map(( + [name, o], + ) => [toKebabCase(name), o.alias!]), + ), + // Marks the flags that are not passed, so they can fall back to the env. + default: Object.fromEntries( + options.map(([name]) => [toKebabCase(name), NOT_PASSED]), + ), + "--": true, + unknown: (arg, key) => { + if (key === undefined) positionals.push(arg); + else unknownFlags.push(arg.split("=")[0]!); + return false; + }, + }); + if (unknownFlags.length > 0) { + const names = [ + ...options.map(([name]) => toKebabCase(name)), + HELP.long, + ]; + const described = unknownFlags.map((flag) => { + const match = flag.startsWith("--") + ? suggest(flag.slice(2), names) + : undefined; + return match ? `${flag} (did you mean --${match}?)` : flag; + }); + throw new UsageError( + `Unknown option${unknownFlags.length > 1 ? "s" : ""}: ${ + described.join(", ") + }`, + ); + } + + const flags: Record = {}; + for (const [name, option] of options) { + const flag = toKebabCase(name); + let value = (parsed as Record)[flag]; + if (value === NOT_PASSED) { + const fromEnv = option.env ? env(option.env) : undefined; + if (fromEnv) { + value = option.type === "boolean" + ? parseBoolean(fromEnv, option.env!) + : fromEnv; + } else { + value = option.default ?? + (option.type === "boolean" ? false : undefined); + } + } + if (value === undefined && option.type === "string" && option.required) { + throw new UsageError( + `Missing required option --${flag}${ + option.env ? ` (or environment variable ${option.env})` : "" + }`, + ); + } + flags[name] = value; + } + + return { flags, positionals, rest: (parsed["--"] ?? []) as string[] }; +} + +function assign( + command: AnyCommand, + positionals: string[], +): Record { + const args: Record = {}; + const definitions = argsOf(command); + + definitions.forEach(({ name, required, variadic }, index) => { + const missing = variadic + ? positionals.length <= index + : positionals[index] === undefined; + if (required && missing) { + throw new UsageError(`Missing required argument <${name}>`); + } + args[name] = variadic ? positionals.slice(index) : positionals[index]; + }); + + const accepted = definitions.some((a) => a.variadic) + ? Infinity + : definitions.length; + if (positionals.length > accepted) { + throw new UsageError(`Unexpected argument '${positionals[accepted]}'`); + } + return args; +} + +function closest( + input: string, + candidates: string[], + maxDistance: number, +): string | undefined { + if (candidates.length === 0) return undefined; + const match = closestString(input, candidates); + const distance = levenshteinDistance( + input.toLowerCase(), + match.toLowerCase(), + ); + return distance <= maxDistance ? match : undefined; +} + +function parseBoolean(value: string, name: string): boolean { + const normalized = value.toLowerCase(); + if (TRUE.includes(normalized)) return true; + if (FALSE.includes(normalized)) return false; + throw new UsageError( + `Environment variable ${name} is '${value}', expected one of ${ + [...TRUE, ...FALSE].join(", ") + }`, + ); +} + +function readEnv(name: string): string | undefined { + try { + return Deno.env.get(name); + } catch (error) { + if (error instanceof Deno.errors.NotCapable) return undefined; + throw error; + } +} + +function table(rows: Array<[string, string]>): string { + const width = Math.max(...rows.map(([left]) => left.length)); + return rows.map(([left, right]) => + ` ${left.padEnd(width)}${right ? ` ${right}` : ""}`.trimEnd() + ).join("\n"); +} diff --git a/cli/deno.json b/cli/deno.json new file mode 100644 index 0000000..09b42be --- /dev/null +++ b/cli/deno.json @@ -0,0 +1,9 @@ +{ + "version": "0.0.1", + "name": "@stdext/cli", + "exports": { + ".": "./mod.ts", + "./parse-args": "./parse_args.ts", + "./command": "./command.ts" + } +} diff --git a/cli/mod.ts b/cli/mod.ts new file mode 100644 index 0000000..eb55d26 --- /dev/null +++ b/cli/mod.ts @@ -0,0 +1,2 @@ +export * from "./parse_args.ts"; +export * from "./command.ts"; diff --git a/cli/parse_args.test.ts b/cli/parse_args.test.ts new file mode 100644 index 0000000..ca0dbc0 --- /dev/null +++ b/cli/parse_args.test.ts @@ -0,0 +1,193 @@ +import { assertEquals, assertThrows } from "@std/assert"; +import { parseArgs as stdParseArgs } from "@std/cli/parse-args"; +import { parseArgs } from "./parse_args.ts"; + +Deno.test("parseArgs matches @std/cli without required", async (t) => { + const cases: [string, string[], Parameters[1]][] = [ + ["no options", ["a", "--b", "c", "-d"], undefined], + ["boolean and string", ["--v", "--n", "x", "rest"], { + boolean: ["v"], + string: ["n"], + }], + ["alias and default", ["-n", "x"], { + alias: { n: "name" }, + default: { port: 80 }, + }], + ["collect and negatable", ["--t", "a", "--t", "b", "--no-c"], { + collect: ["t"], + negatable: ["c"], + boolean: ["c"], + }], + ["double dash", ["a", "--", "b"], { "--": true }], + ["stop early", ["a", "--b"], { stopEarly: true }], + ["empty required", ["--a"], undefined], + ]; + for (const [name, args, options] of cases) { + await t.step(name, () => { + assertEquals(parseArgs(args, options), stdParseArgs(args, options)); + assertEquals( + parseArgs(args, { ...options, required: [] }), + stdParseArgs(args, options), + ); + }); + } +}); + +Deno.test("parseArgs required", async (t) => { + await t.step("passes when present", () => { + assertEquals( + parseArgs(["--name", "foo", "x"], { + string: ["name"], + required: ["name"], + }), + { name: "foo", _: ["x"] }, + ); + }); + + await t.step("throws when missing", () => { + assertThrows( + () => parseArgs(["x"], { required: ["name"] }), + TypeError, + "Missing required flag: --name", + ); + }); + + await t.step("lists all missing flags", () => { + assertThrows( + () => parseArgs(["--bb"], { required: ["aa", "bb", "cc", "dd"] }), + TypeError, + "Missing required flags: --aa, --cc, --dd", + ); + }); + + await t.step("short flags are shown with a single dash", () => { + assertThrows( + () => parseArgs([], { required: ["n"] }), + TypeError, + "Missing required flag: -n", + ); + }); + + await t.step("an alias satisfies the flag", () => { + const options = { alias: { n: "name" }, string: ["name"] }; + assertEquals( + parseArgs(["-n", "x"], { ...options, required: ["name"] }).name, + "x", + ); + assertEquals( + parseArgs(["--name", "x"], { ...options, required: ["n"] }).n, + "x", + ); + assertThrows( + () => parseArgs([], { ...options, required: ["n"] }), + TypeError, + "-n", + ); + }); + + await t.step("boolean flags", () => { + const options = { required: ["verbose"] }; + assertEquals(parseArgs(["--verbose"], options).verbose, true); + assertThrows(() => parseArgs([], options), TypeError, "--verbose"); + // Flags declared as boolean always have a value. + assertEquals( + parseArgs([], { boolean: ["verbose"], required: ["verbose"] }).verbose, + false, + ); + }); + + await t.step("negated flags are present", () => { + assertEquals( + parseArgs(["--no-color"], { + boolean: ["color"], + negatable: ["color"], + required: ["color"], + }).color, + false, + ); + }); + + await t.step("falsy values are present", () => { + assertEquals( + parseArgs(["--n=0", "--s="], { required: ["n", "s"] }).n, + 0, + ); + }); + + await t.step("defaults satisfy a required flag", () => { + assertEquals( + parseArgs([], { default: { port: 80 }, required: ["port"] }).port, + 80, + ); + }); + + await t.step("other defaults are kept", () => { + assertEquals( + parseArgs(["--a"], { default: { b: 2 }, required: ["a"] }), + { a: true, b: 2, _: [] }, + ); + }); + + await t.step("flags after the double dash do not count", () => { + assertThrows( + () => parseArgs(["--", "--name"], { "--": true, required: ["name"] }), + TypeError, + ); + }); +}); + +Deno.test("parseArgs infers the result from the options like @std/cli", () => { + const args = parseArgs(["--name", "foo"], { + string: ["name", "other"], + boolean: ["verbose"], + alias: { name: "n" }, + default: { port: 80 }, + }); + + const name: string | undefined = args.name; + const alias: string | undefined = args.n; + const verbose: boolean = args.verbose; + const port: NonNullable = args.port; + const other: string | undefined = args.other; + // @ts-expect-error `other` is not required and has no default + const definitelyOther: string = args.other; + // @ts-expect-error `verbose` is a boolean + const verboseString: string = args.verbose; + + assertEquals( + [name, alias, verbose, port, other, definitelyOther, verboseString], + ["foo", "foo", false, 80, undefined, undefined, false], + ); +}); + +Deno.test("parseArgs types required and default flags as defined", () => { + const args = parseArgs(["--name", "foo", "--other", "x", "--undeclared"], { + string: ["name", "other", "optional", "withDefault"], + boolean: ["verbose"], + alias: { name: "n", other: ["o", "oo"] }, + default: { withDefault: "d", extra: 1 }, + required: ["name", "o", "undeclared"], + }); + + const name: string = args.name; + const nameAlias: string = args.n; + const other: string = args.other; + const otherAlias: string = args.oo; + const withDefault: string = args.withDefault; + const extra: NonNullable = args.extra; + const undeclared: NonNullable = args.undeclared; + const underscore: Array = args._; + const optional: string | undefined = args.optional; + // @ts-expect-error `optional` is not required, and can be undefined + const optionalString: string = args.optional; + + assertEquals( + [name, nameAlias, other, otherAlias, withDefault, extra, underscore], + ["foo", "foo", "x", "x", "d", 1, []], + ); + assertEquals([undeclared, optional, optionalString], [ + true, + undefined, + undefined, + ]); +}); diff --git a/cli/parse_args.ts b/cli/parse_args.ts new file mode 100644 index 0000000..6ccae94 --- /dev/null +++ b/cli/parse_args.ts @@ -0,0 +1,169 @@ +import { + type Args, + parseArgs as stdParseArgs, + type ParseOptions, +} from "@std/cli/parse-args"; +import type { + Aliases, + BooleanType, + Collectable, + Negatable, + StringType, + Values, +} from "./_types.ts"; + +type AsUnion = TValue extends ReadonlyArray ? TItem + : TValue; + +// The flags, and everything they are an alias of or have as an alias. +type WithAliases = TAliases extends + Record ? + | TFlags + | { + [TName in keyof TAliases & string]: [ + Extract>, + ] extends [never] ? never + : TName | AsUnion; + }[keyof TAliases & string] + : TFlags; + +type FlagsOf = Extract< + WithAliases< + | (TDefaults extends Record ? keyof TDefaults & string + : never) + | (TRequired extends ReadonlyArray ? TRequired[number] : never), + TAliases + >, + string +>; + +// The result, where the given flags are not `undefined`. +type WithFlags = string extends TFlags ? TResult + : + & { + [TKey in keyof TResult as TKey extends TFlags ? never : TKey]: + TResult[TKey]; + } + & { + [TKey in TFlags]-?: TKey extends keyof TResult + ? unknown extends TResult[TKey] ? NonNullable + : Exclude + : NonNullable; + }; + +/** + * Options for {@linkcode parseArgs}, which are the options of `parseArgs` in + * `@std/cli` and `required`. + */ +export interface ParseArgsOptions< + TBooleans extends BooleanType = BooleanType, + TStrings extends StringType = StringType, + TCollectable extends Collectable = Collectable, + TNegatable extends Negatable = Negatable, + TDefault extends Record | undefined = + | Record + | undefined, + TAliases extends Aliases | undefined = Aliases | undefined, + TDoubleDash extends boolean | undefined = boolean | undefined, + TRequired extends ReadonlyArray | undefined = + | ReadonlyArray + | undefined, +> extends + ParseOptions< + TBooleans, + TStrings, + TCollectable, + TNegatable, + TDefault, + TAliases, + TDoubleDash + > { + /** + * A list of flags that must have a value in the parsed arguments. + * {@linkcode parseArgs} throws if any of them is missing. A flag is present + * when it, or one of its aliases, is passed, including a negated flag such as + * `--no-color`. A default also counts as a value, and so does the `false` + * that is set for flags in `boolean`. + * + * The flags are not `undefined` in the type of the result, as are the flags + * with a default. + */ + required?: TRequired; +} + +/** + * Parses command line arguments like `parseArgs` in `@std/cli`, and throws when + * a flag marked as `required` is not present. + * + * It is a drop in replacement for the `@std/cli` version, and any options + * and behavior not mentioned here are the same. + * + * @param args the command line arguments to parse + * @param options the {@linkcode ParseArgsOptions} to parse with + * @returns the parsed arguments + * @throws {TypeError} when a required flag is missing + * + * @example + * ```ts + * import { parseArgs } from "@stdext/cli/parse-args"; + * import { assertEquals, assertThrows } from "@std/assert"; + * + * const options = { string: ["name"], boolean: ["verbose"], required: ["name"] }; + * + * assertEquals(parseArgs(["--name", "foo"], options).name, "foo"); + * assertThrows(() => parseArgs(["--verbose"], options), TypeError); + * ``` + */ +export function parseArgs< + TArgs extends Values< + TBooleans, + TStrings, + TCollectable, + TNegatable, + TDefaults, + TAliases + >, + TDoubleDash extends boolean | undefined = undefined, + TBooleans extends BooleanType = undefined, + TStrings extends StringType = undefined, + TCollectable extends Collectable = undefined, + TNegatable extends Negatable = undefined, + TDefaults extends Record | undefined = undefined, + TAliases extends Aliases | undefined = undefined, + TAliasArgNames extends string = string, + TAliasNames extends string = string, + const TRequired extends ReadonlyArray | undefined = undefined, +>( + args: readonly string[], + options?: ParseArgsOptions< + TBooleans, + TStrings, + TCollectable, + TNegatable, + TDefaults, + TAliases, + TDoubleDash, + TRequired + >, +): WithFlags< + Args, + FlagsOf +> { + const { required = [], ...parseOptions } = options ?? {}; + const result = stdParseArgs(args, parseOptions as ParseOptions) as Record< + string, + unknown + >; + + const missing = required.filter((flag) => result[flag] === undefined); + if (missing.length > 0) { + throw new TypeError( + `Missing required flag${missing.length > 1 ? "s" : ""}: ${ + missing.map((flag) => flag.length === 1 ? `-${flag}` : `--${flag}`) + .join(", ") + }`, + ); + } + + return result as never; +} diff --git a/deno.json b/deno.json index 5f60daf..97a8f9f 100644 --- a/deno.json +++ b/deno.json @@ -13,6 +13,8 @@ "@std/io": "jsr:@std/io@^0", "@std/json": "jsr:@std/json@^1", "@std/path": "jsr:@std/path@^1", + "@std/streams": "jsr:@std/streams@^1", + "@std/text": "jsr:@std/text@^1", "@std/toml": "jsr:@std/toml@^1", "@std/xml": "jsr:@std/xml@^0", "@deno/wasmbuild": "jsr:@deno/wasmbuild@^0", @@ -33,6 +35,7 @@ }, "workspace": [ "./assert", + "./cli", "./collections", "./crypto", "./database", @@ -45,7 +48,8 @@ "./lexer", "./types", "./validation", - "./xml" + "./xml", + "./zip" ], "exclude": [ "./_wasm/target", diff --git a/ffi/dlopen.test.ts b/ffi/dlopen.test.ts index 7638c80..ef6a02b 100644 --- a/ffi/dlopen.test.ts +++ b/ffi/dlopen.test.ts @@ -9,9 +9,9 @@ import { import { join } from "@std/path"; const TEST_FILE_URL = - "https://raw.githubusercontent.com/halvardssm/stdext/refs/tags/0.0.1/README.md"; + "https://raw.githubusercontent.com/halvardssm/std-js/refs/tags/0.0.1/README.md"; const TEST_ZIP_URL = - "https://github.com/halvardssm/stdext/archive/refs/tags/0.0.1.zip"; + "https://github.com/halvardssm/std-js/archive/refs/tags/0.0.1.zip"; // `Deno.build` is read-only in the typings, and the tests mock the os // detection by replacing it. @@ -78,7 +78,7 @@ Deno.test("cacheRemoteFile", async (t) => { const result = await cacheRemoteFile({ url: TEST_ZIP_URL, type: "zip", - archivePath: "stdext-0.0.1/README.md", + archivePath: "std-js-0.0.1/README.md", }, { path: testCacheDir, cacheControl: "reload", @@ -97,7 +97,7 @@ Deno.test("cacheRemoteFile", async (t) => { const result = await cacheRemoteFile({ url: TEST_ZIP_URL, type: "zip", - archivePath: "stdext-0.0.1/README.md", + archivePath: "std-js-0.0.1/README.md", }, { path: testCacheDir, }); @@ -117,14 +117,14 @@ Deno.test("cacheRemoteFile", async (t) => { const first = await cacheRemoteFile({ url: TEST_ZIP_URL, type: "zip", - archivePath: "stdext-0.0.1/README.md", + archivePath: "std-js-0.0.1/README.md", }, { path: testCacheDir, }); const second = await cacheRemoteFile({ url: TEST_ZIP_URL, type: "zip", - archivePath: "stdext-0.0.1/README.md", + archivePath: "std-js-0.0.1/README.md", }, { path: testCacheDir, }); @@ -213,12 +213,12 @@ Deno.test("getCachePath", async (t) => { Deno.test("urlToPathSegments", () => { const url = new URL( - "https://user:pass@sub.github.com:123/halvardssm/stdext/blob/0.0.1/README.md?t=a&b=2#asdf", + "https://user:pass@sub.github.com:123/halvardssm/std-js/blob/0.0.1/README.md?t=a&b=2#asdf", ); const result = urlToPathSegments(url); assertEquals( result, - "https/sub.github.com/halvardssm/stdext/blob/0.0.1/README.md", + "https/sub.github.com/halvardssm/std-js/blob/0.0.1/README.md", ); }); diff --git a/ffi/dlopen.ts b/ffi/dlopen.ts index 5b23a0e..2cf80e3 100644 --- a/ffi/dlopen.ts +++ b/ffi/dlopen.ts @@ -1,6 +1,7 @@ import { ensureDir, exists } from "@std/fs"; import { cacheFile, type CacheFileOptions, denoCacheDir } from "@stdext/fs"; import { dirname, join, resolve } from "@std/path"; +import { unzipDir } from "@stdext/zip/unzip"; /** * Options for dlopen function @@ -170,7 +171,7 @@ export async function cacheRemoteFile( e.name !== "NotFound" && console.warn(e.message) ); } - await unzip(absoluteFilePath, unarchivedPath); + await unzipDir(absoluteFilePath, unarchivedPath); } absoluteFilePath = join(unarchivedPath, fileOptions.archivePath!); const fileExistsInArchive = await exists(absoluteFilePath); @@ -223,17 +224,3 @@ export function urlToPathSegments(url: URL): string { export function withoutExt(path: string | URL): string { return path.toString().replace(/\.+\w+$/, ""); } - -async function unzip(src: string, dest: string): Promise { - const cmd = new Deno.Command("unzip", { - args: [src, "-d", dest], - }); - - const output = await cmd.output(); - - if (!output.success) { - throw new Error( - `Failed to unzip:\n${new TextDecoder().decode(output.stderr)}`, - ); - } -} diff --git a/fs/cache.test.ts b/fs/cache.test.ts index 826a458..5f0e7c2 100644 --- a/fs/cache.test.ts +++ b/fs/cache.test.ts @@ -4,7 +4,7 @@ import { exists } from "@std/fs"; import { join } from "@std/path"; const TEST_FILE_URL = - "https://raw.githubusercontent.com/halvardssm/stdext/refs/tags/0.0.1/README.md"; + "https://raw.githubusercontent.com/halvardssm/std-js/refs/tags/0.0.1/README.md"; // `Deno.build` is read-only in the typings, and the tests mock the os // detection by replacing it. @@ -273,7 +273,7 @@ Deno.test("cacheFile", async (t) => { "throws error when remote file cannot be fetched", async () => { const invalidUrl = - "https://raw.githubusercontent.com/halvardssm/stdext/refs/tags/0.0.1/NONEXISTENT.md"; + "https://raw.githubusercontent.com/halvardssm/std-js/refs/tags/0.0.1/NONEXISTENT.md"; const invalidPath = join(testCacheDir, "NONEXISTENT.md"); await assertRejects( diff --git a/mise.toml b/mise.toml index 98dd712..fdfcdf7 100644 --- a/mise.toml +++ b/mise.toml @@ -4,11 +4,22 @@ rust = { version = "latest", components = ["rustfmt"], targets = [ "wasm32-unknown-unknown", ] } +[settings] +jobs = 1 + [tasks.check] description = "Format check, lint, and type-check" depends = ["format:check", "lint"] run = "deno check **/*.ts" +[tasks."check:pre-commit"] +description = "Fast checks to run before a commit: format check, lint, and type-check" +depends = ["check", "build:wasm:check", "test", "publish:dry:dirty"] + +[tasks."check:pre-push"] +description = "Full checks to run before a push: pre-commit checks, wasm parity, and tests" +depends = ["check:pre-commit", "build:wasm:check", "test", "publish:dry"] + [tasks.lint] description = "Lint the TypeScript code" run = "deno lint" @@ -59,3 +70,16 @@ run = "cargo fmt --all -- --check" description = "Format (JS + Rust) and auto-fix lint rules" depends = ["format"] run = "deno lint --fix" + +[tasks.publish] +description = "Publish the workspace packages to JSR" +confirm = "Publish all workspace packages to JSR?" +run = "deno publish" + +[tasks."publish:dry"] +description = "Check that the workspace packages can be published to JSR" +run = "deno publish --dry-run" + +[tasks."publish:dry:dirty"] +description = "Check that the workspace packages can be published to JSR" +run = "deno publish --dry-run --allow-dirty" diff --git a/zip/README.md b/zip/README.md new file mode 100644 index 0000000..23e1e12 --- /dev/null +++ b/zip/README.md @@ -0,0 +1,23 @@ +# @stdext/zip + +The zip package contains helpers for creating and extracting zip archives, with +no dependencies other than the Deno runtime. + +- `zip` and `unzip` work on in-memory archives. +- `zipDir` and `unzipDir` work on the file system. + +Archives are written with the stored and deflate methods, and read with the same +two, which covers practically all archives in use. Encryption and ZIP64 (more +than 65535 entries, or sizes above 4 GiB) are not supported, and throw. + +```ts +import { unzip, zip } from "@stdext/zip"; + +const archive = await zip([ + { path: "hello.txt", data: "Hello, world!" }, + { path: "bin/run.sh", data: "#!/bin/sh\n", mode: 0o755 }, +]); + +const entries = await unzip(archive); +console.log(entries.map((entry) => entry.path)); // ["hello.txt", "bin/run.sh"] +``` diff --git a/zip/_utils.ts b/zip/_utils.ts new file mode 100644 index 0000000..8c8eb64 --- /dev/null +++ b/zip/_utils.ts @@ -0,0 +1,32 @@ +import { toBytes } from "@std/streams/to-bytes"; + +export const LOCAL_SIGNATURE = 0x04034b50; +export const CENTRAL_SIGNATURE = 0x02014b50; +export const END_SIGNATURE = 0x06054b50; +export const LOCAL_HEADER_SIZE = 30; +export const CENTRAL_HEADER_SIZE = 46; +export const END_SIZE = 22; +export const METHOD_STORED = 0; +export const METHOD_DEFLATE = 8; +export const MAX_U16 = 0xffff; +export const MAX_U32 = 0xffffffff; + +const CRC_TABLE = new Uint32Array(256).map((_, n) => { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + return c >>> 0; +}); + +export function crc32(data: Uint8Array): number { + let crc = 0xffffffff; + for (const byte of data) crc = CRC_TABLE[(crc ^ byte) & 0xff]! ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +export async function transform( + data: Uint8Array, + stream: CompressionStream | DecompressionStream, +): Promise { + const source = new Blob([data as BlobPart]).stream(); + return await toBytes(source.pipeThrough(stream)); +} diff --git a/zip/deno.json b/zip/deno.json new file mode 100644 index 0000000..8311544 --- /dev/null +++ b/zip/deno.json @@ -0,0 +1,9 @@ +{ + "version": "0.0.1", + "name": "@stdext/zip", + "exports": { + ".": "./mod.ts", + "./zip": "./zip.ts", + "./unzip": "./unzip.ts" + } +} diff --git a/zip/fs.ts b/zip/fs.ts new file mode 100644 index 0000000..feb25b3 --- /dev/null +++ b/zip/fs.ts @@ -0,0 +1,152 @@ +import { ensureDir, walk } from "@std/fs"; +import { + dirname, + isAbsolute, + join, + relative, + resolve, + SEPARATOR, +} from "@std/path"; +import { unzip, type UnzipEntry } from "./unzip.ts"; +import { zip, type ZipEntry } from "./zip.ts"; + +const FILE_TYPE_MASK = 0o170000; +const FILE_TYPE_SYMLINK = 0o120000; +const PERMISSION_MASK = 0o777; + +/** + * Zips a directory into an archive file + * + * The contents of the directory are added with paths relative to it, including + * empty directories and, on Unix, permissions. Symbolic links are skipped. + * + * @param source the directory to zip + * @param destination the path of the archive file to write + * + * @example + * ```ts ignore + * import { zipDir } from "@stdext/zip/fs"; + * + * await zipDir("./dist", "./dist.zip"); + * ``` + */ +export async function zipDir( + source: string, + destination: string, +): Promise { + const entries: ZipEntry[] = []; + const root = resolve(source); + + for await (const walked of walk(root, { includeSymlinks: false })) { + if (walked.path === root) continue; + const path = relative(root, walked.path).split(SEPARATOR).join("/"); + const info = await Deno.stat(walked.path); + const mode = info.mode === null ? undefined : info.mode & PERMISSION_MASK; + const lastModified = info.mtime ?? undefined; + + if (walked.isDirectory) { + entries.push({ path: `${path}/`, mode, lastModified }); + } else { + entries.push({ + path, + data: await Deno.readFile(walked.path), + mode, + lastModified, + }); + } + } + + await Deno.writeFile(destination, await zip(entries)); +} + +/** + * Extracts an archive file into a directory + * + * The directory is created if needed, and existing files are overwritten. + * Entries that would be written outside of the directory, including through + * symbolic links, are rejected. Unix permissions and symbolic links are + * restored, except on Windows where symbolic links are written as files. + * + * @param source the path of the archive file + * @param destination the directory to extract into + * @throws {TypeError} when the archive is invalid, or an entry is outside of + * the destination + * + * @example + * ```ts ignore + * import { unzipDir } from "@stdext/zip/fs"; + * + * await unzipDir("./dist.zip", "./dist"); + * ``` + */ +export async function unzipDir( + source: string, + destination: string, +): Promise { + const entries = await unzip(await Deno.readFile(source)); + + await ensureDir(destination); + const root = await Deno.realPath(destination); + + // Symlinks are created last, so that no entry is ever written through one. + const symlinks: UnzipEntry[] = []; + + for (const entry of entries) { + const target = resolve(root, entry.path); + assertInside(root, target, entry.path); + + if (entry.isDirectory) { + await ensureDir(target); + assertInside(root, await Deno.realPath(target), entry.path); + continue; + } + + if (isSymlink(entry) && Deno.build.os !== "windows") { + symlinks.push(entry); + continue; + } + + // The parent is resolved on disk, so symlinks that already exist in the + // destination cannot be used to escape it. + await ensureDir(dirname(target)); + assertInside(root, await Deno.realPath(dirname(target)), entry.path); + + // Replace an existing symlink instead of writing through it. + if ((await Deno.lstat(target).catch(() => undefined))?.isSymlink) { + await Deno.remove(target); + } + await Deno.writeFile(target, entry.data); + if (entry.mode !== undefined && Deno.build.os !== "windows") { + await Deno.chmod(target, entry.mode & PERMISSION_MASK); + } + } + + for (const entry of symlinks) { + const target = resolve(root, entry.path); + const linkTarget = new TextDecoder().decode(entry.data); + if (isAbsolute(linkTarget)) { + throw new TypeError( + `Symlink '${entry.path}' points outside of destination`, + ); + } + assertInside(root, resolve(dirname(target), linkTarget), entry.path); + + const parent = await Deno.realPath(dirname(target)).catch(() => { + throw new TypeError(`Symlink '${entry.path}' has no parent directory`); + }); + assertInside(root, parent, entry.path); + + await Deno.remove(target).catch(() => {}); + await Deno.symlink(linkTarget, target); + } +} + +function isSymlink(entry: UnzipEntry): boolean { + return ((entry.mode ?? 0) & FILE_TYPE_MASK) === FILE_TYPE_SYMLINK; +} + +function assertInside(root: string, path: string, entryPath: string): void { + if (path !== root && !path.startsWith(join(root, SEPARATOR))) { + throw new TypeError(`Entry '${entryPath}' is outside of destination`); + } +} diff --git a/zip/mod.ts b/zip/mod.ts new file mode 100644 index 0000000..40ee3bd --- /dev/null +++ b/zip/mod.ts @@ -0,0 +1,2 @@ +export * from "./zip.ts"; +export * from "./unzip.ts"; diff --git a/zip/unzip.test.ts b/zip/unzip.test.ts new file mode 100644 index 0000000..b6f3877 --- /dev/null +++ b/zip/unzip.test.ts @@ -0,0 +1,219 @@ +import { assertEquals, assertRejects } from "@std/assert"; +import { exists } from "@std/fs"; +import { join } from "@std/path"; +import { unzip, unzipDir } from "./unzip.ts"; +import { zip, zipDir } from "./zip.ts"; + +const text = (data: Uint8Array) => new TextDecoder().decode(data); + +Deno.test("unzip errors", async (t) => { + const archive = await zip([{ path: "a.txt", data: "hello hello hello" }]); + + await t.step("not a zip", async () => { + await assertRejects( + () => unzip(new Uint8Array(100)), + TypeError, + "Not a zip", + ); + await assertRejects(() => unzip(new Uint8Array()), TypeError, "Not a zip"); + }); + + await t.step("corrupt data", async () => { + const corrupt = archive.slice(); + // First byte of the file data, right after the local header and name. + corrupt[30 + "a.txt".length] ^= 0xff; + await assertRejects(() => unzip(corrupt), TypeError, "corrupt"); + }); + + await t.step("crc mismatch on stored entry", async () => { + const stored = await zip([{ path: "a", data: "x" }]); + stored[30 + 1] ^= 0xff; + await assertRejects(() => unzip(stored), TypeError, "corrupt"); + }); + + await t.step("encrypted", async () => { + const encrypted = archive.slice(); + const view = new DataView(encrypted.buffer); + const central = view.getUint32(encrypted.length - 22 + 16, true); + view.setUint16(central + 8, 1, true); + await assertRejects(() => unzip(encrypted), TypeError, "encrypted"); + }); + + await t.step("unsupported method", async () => { + const other = archive.slice(); + const view = new DataView(other.buffer); + const central = view.getUint32(other.length - 22 + 16, true); + view.setUint16(central + 10, 12, true); + await assertRejects(() => unzip(other), TypeError, "method 12"); + }); + + await t.step("zip64", async () => { + const zip64 = archive.slice(); + new DataView(zip64.buffer).setUint16(zip64.length - 22 + 10, 0xffff, true); + await assertRejects(() => unzip(zip64), TypeError, "ZIP64"); + }); + + await t.step("invalid central directory", async () => { + const invalid = archive.slice(); + const view = new DataView(invalid.buffer); + view.setUint32(invalid.length - 22 + 16, 0, true); + await assertRejects(() => unzip(invalid), TypeError, "central directory"); + }); + + await t.step("truncated", async () => { + const truncated = archive.slice(); + const view = new DataView(truncated.buffer); + const central = view.getUint32(truncated.length - 22 + 16, true); + view.setUint32(central + 20, 0xfffff, true); + await assertRejects(() => unzip(truncated), TypeError, "truncated"); + }); +}); + +Deno.test("unzip reads archives made by other tools", async () => { + const dir = await Deno.makeTempDir(); + try { + await Deno.writeTextFile(`${dir}/hello.txt`, "hello\n".repeat(50)); + const result = await new Deno.Command("zip", { + args: ["-q", "-r", "out.zip", "hello.txt"], + cwd: dir, + stderr: "null", + }).output().catch(() => undefined); + if (!result?.success) return; // zip binary not available + const [entry] = await unzip(await Deno.readFile(`${dir}/out.zip`)); + assertEquals(entry!.path, "hello.txt"); + assertEquals(text(entry!.data), "hello\n".repeat(50)); + } finally { + await Deno.remove(dir, { recursive: true }); + } +}); + +const isWindows = Deno.build.os === "windows"; + +async function withTempDir(fn: (dir: string) => Promise) { + const dir = await Deno.realPath(await Deno.makeTempDir()); + try { + await fn(dir); + } finally { + await Deno.remove(dir, { recursive: true }); + } +} + +Deno.test("zipDir and unzipDir roundtrip", async () => { + await withTempDir(async (dir) => { + const src = join(dir, "src"); + await Deno.mkdir(join(src, "sub/empty"), { recursive: true }); + await Deno.writeTextFile(join(src, "a.txt"), "a"); + await Deno.writeTextFile(join(src, "sub/b.txt"), "b".repeat(500)); + await Deno.writeTextFile(join(src, "run.sh"), "#!/bin/sh"); + if (!isWindows) { + await Deno.chmod(join(src, "run.sh"), 0o755); + await Deno.symlink("a.txt", join(src, "link")); + } + + await zipDir(src, join(dir, "out.zip")); + const paths = (await unzip(await Deno.readFile(join(dir, "out.zip")))) + .map((e) => e.path).sort(); + assertEquals(paths, ["a.txt", "run.sh", "sub/", "sub/b.txt", "sub/empty/"]); + + const dest = join(dir, "nested/dest"); + await unzipDir(join(dir, "out.zip"), dest); + assertEquals(await Deno.readTextFile(join(dest, "a.txt")), "a"); + assertEquals( + await Deno.readTextFile(join(dest, "sub/b.txt")), + "b".repeat(500), + ); + assertEquals((await Deno.stat(join(dest, "sub/empty"))).isDirectory, true); + if (!isWindows) { + assertEquals( + (await Deno.stat(join(dest, "run.sh"))).mode! & 0o777, + 0o755, + ); + } + }); +}); + +Deno.test("unzipDir overwrites existing files", async () => { + await withTempDir(async (dir) => { + await Deno.writeFile( + join(dir, "in.zip"), + await zip([ + { path: "a.txt", data: "new" }, + ]), + ); + await Deno.writeTextFile(join(dir, "a.txt"), "old"); + await unzipDir(join(dir, "in.zip"), dir); + assertEquals(await Deno.readTextFile(join(dir, "a.txt")), "new"); + }); +}); + +Deno.test("unzipDir restores symlinks", { + ignore: isWindows, +}, async () => { + await withTempDir(async (dir) => { + const mode = 0o120000 | 0o777; + await Deno.writeFile( + join(dir, "in.zip"), + await zip([ + { path: "lib.so.1", data: "lib" }, + { path: "lib.so", data: "lib.so.1", mode }, + { path: "up/" }, + { path: "up/link", data: "../lib.so.1", mode }, + ]), + ); + await unzipDir(join(dir, "in.zip"), join(dir, "out")); + assertEquals(await Deno.readLink(join(dir, "out/lib.so")), "lib.so.1"); + assertEquals(await Deno.readTextFile(join(dir, "out/up/link")), "lib"); + }); +}); + +Deno.test("unzipDir rejects unsafe entries", async (t) => { + const cases: Record[0]> = { + "parent path": [{ path: "../evil.txt", data: "x" }], + "nested parent path": [{ path: "a/../../evil.txt", data: "x" }], + "absolute path": [{ path: "/tmp/evil.txt", data: "x" }], + "parent directory": [{ path: "../evil/" }], + }; + if (!isWindows) { + const mode = 0o120777; + Object.assign(cases, { + "symlink outside": [{ path: "l", data: "../outside", mode }], + "absolute symlink": [{ path: "l", data: "/etc", mode }], + "symlink without parent": [{ path: "x/l", data: "a", mode }], + }); + } + + for (const [name, entries] of Object.entries(cases)) { + await t.step(name, async () => { + await withTempDir(async (dir) => { + await Deno.writeFile(join(dir, "in.zip"), await zip(entries)); + await assertRejects( + () => unzipDir(join(dir, "in.zip"), join(dir, "out")), + TypeError, + ); + assertEquals(await exists(join(dir, "evil.txt")), false); + }); + }); + } + + await t.step({ + name: "write through a symlinked directory", + ignore: isWindows, + fn: async () => { + await withTempDir(async (dir) => { + await Deno.mkdir(join(dir, "out")); + await Deno.mkdir(join(dir, "outside")); + await Deno.symlink(join(dir, "outside"), join(dir, "out/link")); + await Deno.writeFile( + join(dir, "in.zip"), + await zip([{ path: "link/evil.txt", data: "x" }]), + ); + await assertRejects( + () => unzipDir(join(dir, "in.zip"), join(dir, "out")), + TypeError, + "outside of destination", + ); + assertEquals(await exists(join(dir, "outside/evil.txt")), false); + }); + }, + }); +}); diff --git a/zip/unzip.ts b/zip/unzip.ts new file mode 100644 index 0000000..08c8523 --- /dev/null +++ b/zip/unzip.ts @@ -0,0 +1,274 @@ +import { ensureDir } from "@std/fs"; +import { dirname, isAbsolute, join, resolve, SEPARATOR } from "@std/path"; +import { + CENTRAL_HEADER_SIZE, + CENTRAL_SIGNATURE, + crc32, + END_SIGNATURE, + END_SIZE, + LOCAL_HEADER_SIZE, + LOCAL_SIGNATURE, + MAX_U16, + MAX_U32, + METHOD_DEFLATE, + METHOD_STORED, + transform, +} from "./_utils.ts"; + +/** + * An entry read from an archive with {@linkcode unzip} + */ +export type UnzipEntry = { + /** Path of the entry within the archive, using `/` as separator */ + path: string; + /** Uncompressed content of the entry, empty for directories */ + data: Uint8Array; + /** Whether the entry is a directory */ + isDirectory: boolean; + /** Unix permission and file type bits, if the archive was made on Unix */ + mode?: number; + /** Last modified time */ + lastModified: Date; +}; + +const MAX_COMMENT_SIZE = 0xffff; +const FLAG_ENCRYPTED = 1 << 0; +const HOST_UNIX = 3; +const FILE_TYPE_MASK = 0o170000; +const FILE_TYPE_SYMLINK = 0o120000; +const PERMISSION_MASK = 0o777; + +const decoder = new TextDecoder(); + +function fromDosDateTime(date: number, time: number): Date { + return new Date( + ((date >> 9) & 0x7f) + 1980, + Math.max(((date >> 5) & 0xf) - 1, 0), + Math.max(date & 0x1f, 1), + (time >> 11) & 0x1f, + (time >> 5) & 0x3f, + (time & 0x1f) * 2, + ); +} + +/** + * Reads a zip archive + * + * Entry contents are verified against the CRC-32 stored in the archive. The + * archive is read in memory, and entry paths are returned as stored: callers + * extracting to disk need to validate them. + * + * @param archive the zip archive + * @returns the entries of the archive, in archive order + * @throws {TypeError} when the archive is invalid, encrypted, uses ZIP64 or a + * compression method other than stored and deflate, or an entry is corrupt + * + * @example + * ```ts + * import { unzip } from "@stdext/zip/unzip"; + * import { zip } from "@stdext/zip/zip"; + * import { assertEquals } from "@std/assert"; + * + * const archive = await zip([{ path: "hello.txt", data: "Hello, world!" }]); + * const [entry] = await unzip(archive); + * + * assertEquals(entry!.path, "hello.txt"); + * assertEquals(new TextDecoder().decode(entry!.data), "Hello, world!"); + * ``` + */ +export async function unzip(archive: Uint8Array): Promise { + const view = new DataView( + archive.buffer, + archive.byteOffset, + archive.byteLength, + ); + + let end = -1; + const lowest = Math.max(archive.length - END_SIZE - MAX_COMMENT_SIZE, 0); + for (let i = archive.length - END_SIZE; i >= lowest; i--) { + if (view.getUint32(i, true) === END_SIGNATURE) { + end = i; + break; + } + } + if (end < 0) throw new TypeError("Not a zip archive"); + + const count = view.getUint16(end + 10, true); + let position = view.getUint32(end + 16, true); + if (count === MAX_U16 || position === MAX_U32) { + throw new TypeError("ZIP64 archives are not supported"); + } + + const entries: UnzipEntry[] = []; + for (let i = 0; i < count; i++) { + if ( + position + CENTRAL_HEADER_SIZE > archive.length || + view.getUint32(position, true) !== CENTRAL_SIGNATURE + ) { + throw new TypeError("Invalid zip central directory"); + } + + const madeByHost = view.getUint8(position + 5); + const flags = view.getUint16(position + 8, true); + const method = view.getUint16(position + 10, true); + const time = view.getUint16(position + 12, true); + const date = view.getUint16(position + 14, true); + const crc = view.getUint32(position + 16, true); + const compressedSize = view.getUint32(position + 20, true); + const size = view.getUint32(position + 24, true); + const nameLength = view.getUint16(position + 28, true); + const extraLength = view.getUint16(position + 30, true); + const commentLength = view.getUint16(position + 32, true); + const externalAttributes = view.getUint32(position + 38, true); + const localOffset = view.getUint32(position + 42, true); + const nameStart = position + CENTRAL_HEADER_SIZE; + const path = decoder.decode( + archive.subarray(nameStart, nameStart + nameLength), + ); + position = nameStart + nameLength + extraLength + commentLength; + + if (flags & FLAG_ENCRYPTED) { + throw new TypeError( + `Entry '${path}' is encrypted, which is not supported`, + ); + } + if (compressedSize === MAX_U32 || size === MAX_U32) { + throw new TypeError("ZIP64 archives are not supported"); + } + if ( + localOffset + LOCAL_HEADER_SIZE > archive.length || + view.getUint32(localOffset, true) !== LOCAL_SIGNATURE + ) { + throw new TypeError(`Invalid local header for entry '${path}'`); + } + + const dataStart = localOffset + LOCAL_HEADER_SIZE + + view.getUint16(localOffset + 26, true) + + view.getUint16(localOffset + 28, true); + if (dataStart + compressedSize > archive.length) { + throw new TypeError(`Entry '${path}' is truncated`); + } + const body = archive.subarray(dataStart, dataStart + compressedSize); + + let data: Uint8Array; + if (method === METHOD_STORED) { + data = body.slice(); + } else if (method === METHOD_DEFLATE) { + try { + data = await transform(body, new DecompressionStream("deflate-raw")); + } catch { + throw new TypeError(`Entry '${path}' is corrupt`); + } + } else { + throw new TypeError( + `Entry '${path}' uses unsupported compression method ${method}`, + ); + } + if (data.length !== size || crc32(data) !== crc) { + throw new TypeError(`Entry '${path}' is corrupt`); + } + + const isDirectory = path.endsWith("/"); + entries.push({ + path, + data, + isDirectory, + mode: madeByHost === HOST_UNIX ? externalAttributes >>> 16 : undefined, + lastModified: fromDosDateTime(date, time), + }); + } + + return entries; +} + +/** + * Extracts an archive file into a directory + * + * The directory is created if needed, and existing files are overwritten. + * Entries that would be written outside of the directory, including through + * symbolic links, are rejected. Unix permissions and symbolic links are + * restored, except on Windows where symbolic links are written as files. + * + * @param source the path of the archive file + * @param destination the directory to extract into + * @throws {TypeError} when the archive is invalid, or an entry is outside of + * the destination + * + * @example + * ```ts ignore + * import { unzipDir } from "@stdext/zip/unzip"; + * + * await unzipDir("./dist.zip", "./dist"); + * ``` + */ +export async function unzipDir( + source: string, + destination: string, +): Promise { + const entries = await unzip(await Deno.readFile(source)); + + await ensureDir(destination); + const root = await Deno.realPath(destination); + + // Symlinks are created last, so that no entry is ever written through one. + const symlinks: UnzipEntry[] = []; + + for (const entry of entries) { + const target = resolve(root, entry.path); + assertInside(root, target, entry.path); + + if (entry.isDirectory) { + await ensureDir(target); + assertInside(root, await Deno.realPath(target), entry.path); + continue; + } + + if (isSymlink(entry) && Deno.build.os !== "windows") { + symlinks.push(entry); + continue; + } + + // The parent is resolved on disk, so symlinks that already exist in the + // destination cannot be used to escape it. + await ensureDir(dirname(target)); + assertInside(root, await Deno.realPath(dirname(target)), entry.path); + + // Replace an existing symlink instead of writing through it. + if ((await Deno.lstat(target).catch(() => undefined))?.isSymlink) { + await Deno.remove(target); + } + await Deno.writeFile(target, entry.data); + if (entry.mode !== undefined && Deno.build.os !== "windows") { + await Deno.chmod(target, entry.mode & PERMISSION_MASK); + } + } + + for (const entry of symlinks) { + const target = resolve(root, entry.path); + const linkTarget = new TextDecoder().decode(entry.data); + if (isAbsolute(linkTarget)) { + throw new TypeError( + `Symlink '${entry.path}' points outside of destination`, + ); + } + assertInside(root, resolve(dirname(target), linkTarget), entry.path); + + const parent = await Deno.realPath(dirname(target)).catch(() => { + throw new TypeError(`Symlink '${entry.path}' has no parent directory`); + }); + assertInside(root, parent, entry.path); + + await Deno.remove(target).catch(() => {}); + await Deno.symlink(linkTarget, target); + } +} + +function isSymlink(entry: UnzipEntry): boolean { + return ((entry.mode ?? 0) & FILE_TYPE_MASK) === FILE_TYPE_SYMLINK; +} + +function assertInside(root: string, path: string, entryPath: string): void { + if (path !== root && !path.startsWith(join(root, SEPARATOR))) { + throw new TypeError(`Entry '${entryPath}' is outside of destination`); + } +} diff --git a/zip/zip.test.ts b/zip/zip.test.ts new file mode 100644 index 0000000..90b4bbe --- /dev/null +++ b/zip/zip.test.ts @@ -0,0 +1,75 @@ +import { assertEquals, assertRejects } from "@std/assert"; +import { unzip } from "./unzip.ts"; +import { zip } from "./zip.ts"; + +const text = (data: Uint8Array) => new TextDecoder().decode(data); + +Deno.test("zip and unzip roundtrip", async (t) => { + await t.step("files, directories and metadata", async () => { + const lastModified = new Date(2024, 5, 15, 12, 30, 44); + const archive = await zip([ + { path: "dir/", mode: 0o750, lastModified }, + { path: "dir/a.txt", data: "a".repeat(1000), lastModified }, + { path: "run.sh", data: new Uint8Array([1, 2, 3]), mode: 0o755 }, + { path: "ünï/çödé.txt", data: "unicode" }, + { path: "empty" }, + ]); + + const entries = await unzip(archive); + assertEquals(entries.map((e) => e.path), [ + "dir/", + "dir/a.txt", + "run.sh", + "ünï/çödé.txt", + "empty", + ]); + assertEquals(entries[0]!.isDirectory, true); + assertEquals(entries[0]!.mode! & 0o777, 0o750); + assertEquals(entries[0]!.lastModified, lastModified); + assertEquals(text(entries[1]!.data), "a".repeat(1000)); + assertEquals(entries[2]!.data, new Uint8Array([1, 2, 3])); + assertEquals(entries[2]!.mode! & 0o777, 0o755); + assertEquals(entries[2]!.isDirectory, false); + assertEquals(text(entries[3]!.data), "unicode"); + assertEquals(entries[4]!.data.length, 0); + }); + + await t.step("compresses only when it helps", async () => { + const compressible = await zip([{ path: "a", data: "a".repeat(10000) }]); + assertEquals(compressible.length < 1000, true); + // Method is stored for incompressible and empty data. + const stored = await zip([{ path: "a", data: "x" }]); + assertEquals(new DataView(stored.buffer).getUint16(8, true), 0); + }); + + await t.step("empty archive", async () => { + assertEquals(await unzip(await zip([])), []); + }); + + await t.step("throws on ZIP64 sized names", async () => { + await assertRejects( + () => zip([{ path: "a".repeat(0x10000) }]), + RangeError, + "ZIP64", + ); + }); +}); + +Deno.test("zip output is readable by unzip", async () => { + const dir = await Deno.makeTempDir(); + try { + await Deno.writeFile( + `${dir}/out.zip`, + await zip([{ path: "a/b.txt", data: "content" }]), + ); + const result = await new Deno.Command("unzip", { + args: ["-tq", "out.zip"], + cwd: dir, + stdout: "null", + stderr: "null", + }).output().catch(() => undefined); + if (result) assertEquals(result.success, true); + } finally { + await Deno.remove(dir, { recursive: true }); + } +}); diff --git a/zip/zip.ts b/zip/zip.ts new file mode 100644 index 0000000..69ea022 --- /dev/null +++ b/zip/zip.ts @@ -0,0 +1,220 @@ +import { concat } from "@std/bytes"; +import { walk } from "@std/fs"; +import { relative, resolve, SEPARATOR } from "@std/path"; +import { + CENTRAL_HEADER_SIZE, + CENTRAL_SIGNATURE, + crc32, + END_SIGNATURE, + END_SIZE, + LOCAL_HEADER_SIZE, + LOCAL_SIGNATURE, + MAX_U16, + MAX_U32, + METHOD_DEFLATE, + METHOD_STORED, + transform, +} from "./_utils.ts"; + +/** + * An entry to add to an archive with {@linkcode zip} + */ +export type ZipEntry = { + /** + * Path of the entry within the archive, using `/` as separator. A path + * ending in `/` is a directory. + */ + path: string; + /** Content of the entry. Strings are encoded as UTF-8. Ignored for directories. */ + data?: Uint8Array | string; + /** + * Unix permission bits, e.g. `0o755`. Include the file type bits to store + * other types, e.g. `0o120777` for a symbolic link with the target as data. + */ + mode?: number; + /** Last modified time, defaults to now. Has a 2 second resolution from 1980. */ + lastModified?: Date; +}; + +const FLAG_UTF8 = 1 << 11; +// Version needed to extract (2.0, deflate and directories) and the Unix host +// (3), which makes extractors honor the mode in the external attributes. +const VERSION_NEEDED = 20; +const VERSION_MADE_BY = (3 << 8) | VERSION_NEEDED; +const DOS_DIRECTORY = 0x10; +const PERMISSION_MASK = 0o777; + +const encoder = new TextEncoder(); + +function toDosDateTime(date: Date): { time: number; date: number } { + const year = Math.min(Math.max(date.getFullYear(), 1980), 2107); + return { + time: (date.getHours() << 11) | (date.getMinutes() << 5) | + (date.getSeconds() >> 1), + date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(), + }; +} + +/** + * Creates a zip archive + * + * Entries are deflated, or stored when deflating does not make them smaller. + * The archive is built in memory. + * + * @param entries the entries to add to the archive + * @returns the zip archive + * @throws {RangeError} when the archive would need ZIP64 + * + * @example + * ```ts + * import { zip } from "@stdext/zip/zip"; + * import { assertEquals } from "@std/assert"; + * + * const archive = await zip([ + * { path: "docs/", mode: 0o755 }, + * { path: "docs/hello.txt", data: "Hello, world!" }, + * ]); + * + * assertEquals(archive[0], 0x50); // "PK" + * ``` + */ +export async function zip(entries: Iterable): Promise { + const parts: Uint8Array[] = []; + const central: Uint8Array[] = []; + let offset = 0; + + for (const entry of entries) { + const isDirectory = entry.path.endsWith("/"); + const name = encoder.encode(entry.path); + const raw = isDirectory || entry.data === undefined + ? new Uint8Array() + : typeof entry.data === "string" + ? encoder.encode(entry.data) + : entry.data; + + let method = METHOD_STORED; + let body = raw; + if (raw.length > 0) { + const deflated = await transform( + raw, + new CompressionStream("deflate-raw"), + ); + if (deflated.length < raw.length) { + method = METHOD_DEFLATE; + body = deflated; + } + } + + if (name.length > MAX_U16 || body.length > MAX_U32 || offset > MAX_U32) { + throw new RangeError(`Entry '${entry.path}' requires ZIP64`); + } + + const { time, date } = toDosDateTime(entry.lastModified ?? new Date()); + const crc = crc32(raw); + const mode = entry.mode ?? (isDirectory ? 0o755 : 0o644); + // File type bits are kept when given, e.g. for symbolic links. + const fileType = mode & 0o170000 || (isDirectory ? 0o040000 : 0o100000); + const externalAttributes = ((((mode & 0o7777) | fileType) << 16) | + (isDirectory ? DOS_DIRECTORY : 0)) >>> + 0; + + const local = new Uint8Array(LOCAL_HEADER_SIZE + name.length); + const localView = new DataView(local.buffer); + localView.setUint32(0, LOCAL_SIGNATURE, true); + localView.setUint16(4, VERSION_NEEDED, true); + localView.setUint16(6, FLAG_UTF8, true); + localView.setUint16(8, method, true); + localView.setUint16(10, time, true); + localView.setUint16(12, date, true); + localView.setUint32(14, crc, true); + localView.setUint32(18, body.length, true); + localView.setUint32(22, raw.length, true); + localView.setUint16(26, name.length, true); + local.set(name, LOCAL_HEADER_SIZE); + + const header = new Uint8Array(CENTRAL_HEADER_SIZE + name.length); + const view = new DataView(header.buffer); + view.setUint32(0, CENTRAL_SIGNATURE, true); + view.setUint16(4, VERSION_MADE_BY, true); + view.setUint16(6, VERSION_NEEDED, true); + view.setUint16(8, FLAG_UTF8, true); + view.setUint16(10, method, true); + view.setUint16(12, time, true); + view.setUint16(14, date, true); + view.setUint32(16, crc, true); + view.setUint32(20, body.length, true); + view.setUint32(24, raw.length, true); + view.setUint16(28, name.length, true); + view.setUint32(38, externalAttributes, true); + view.setUint32(42, offset, true); + header.set(name, CENTRAL_HEADER_SIZE); + + parts.push(local, body); + central.push(header); + offset += local.length + body.length; + } + + if (central.length > MAX_U16) { + throw new RangeError("Archive has too many entries, requires ZIP64"); + } + + const centralDirectory = concat(central); + if (offset + centralDirectory.length > MAX_U32) { + throw new RangeError("Archive is too large, requires ZIP64"); + } + + const end = new Uint8Array(END_SIZE); + const endView = new DataView(end.buffer); + endView.setUint32(0, END_SIGNATURE, true); + endView.setUint16(8, central.length, true); + endView.setUint16(10, central.length, true); + endView.setUint32(12, centralDirectory.length, true); + endView.setUint32(16, offset, true); + + return concat([...parts, centralDirectory, end]); +} + +/** + * Zips a directory into an archive file + * + * The contents of the directory are added with paths relative to it, including + * empty directories and, on Unix, permissions. Symbolic links are skipped. + * + * @param source the directory to zip + * @param destination the path of the archive file to write + * + * @example + * ```ts ignore + * import { zipDir } from "@stdext/zip/zip"; + * + * await zipDir("./dist", "./dist.zip"); + * ``` + */ +export async function zipDir( + source: string, + destination: string, +): Promise { + const entries: ZipEntry[] = []; + const root = resolve(source); + + for await (const walked of walk(root, { includeSymlinks: false })) { + if (walked.path === root) continue; + const path = relative(root, walked.path).split(SEPARATOR).join("/"); + const info = await Deno.stat(walked.path); + const mode = info.mode === null ? undefined : info.mode & PERMISSION_MASK; + const lastModified = info.mtime ?? undefined; + + if (walked.isDirectory) { + entries.push({ path: `${path}/`, mode, lastModified }); + } else { + entries.push({ + path, + data: await Deno.readFile(walked.path), + mode, + lastModified, + }); + } + } + + await Deno.writeFile(destination, await zip(entries)); +}