diff --git a/README.md b/README.md index 3c3e8e2..126adf2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # TOON for Visual Studio Code [![Visual Studio Marketplace](https://img.shields.io/visual-studio-marketplace/v/toon-format.toon)](https://marketplace.visualstudio.com/items?itemName=toon-format.toon) -[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-lightgrey)](https://github.com/toon-format/spec/blob/main/SPEC.md) +[![SPEC v4.4](https://img.shields.io/badge/spec-v4.4-lightgrey)](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) Highlights [TOON (Token-Oriented Object Notation)](https://github.com/toon-format/toon) in `.toon` files. TOON is a compact, indentation-based encoding of the JSON data model for LLM input. @@ -34,7 +34,7 @@ Folding follows indentation, and pressing Enter after a line ending in `:` inden ## Specification -Targets [TOON spec v4.1](https://github.com/toon-format/spec/blob/main/SPEC.md). +Targets [TOON spec v4.4](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md). ## Resources diff --git a/package.json b/package.json index 3ac4cea..e56d438 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ }, "devDependencies": { "@antfu/eslint-config": "^9.5.1", - "@toon-format/spec": "^4.1.2", + "@toon-format/spec": "^4.4.0", "@types/node": "^26.6.3", "@vscode/vsce": "^4.0.0", "bumpp": "^12.3.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 085eb06..8c5eddc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^9.5.1 version: 9.5.1(@typescript-eslint/typescript-estree@8.71.0(supports-color@7.2.0)(typescript@6.0.3))(@typescript-eslint/utils@8.71.0(eslint@10.11.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(@vue/compiler-sfc@3.5.25)(eslint@10.11.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(ts-declaration-location@1.0.7(typescript@6.0.3))(typescript@6.0.3)(vitest@5.0.3(@types/node@26.6.3)(vite@8.3.1(@types/node@26.6.3)(jiti@2.7.0)(yaml@2.9.1))) '@toon-format/spec': - specifier: ^4.1.2 - version: 4.1.2 + specifier: ^4.4.0 + version: 4.4.0 '@types/node': specifier: ^26.6.3 version: 26.6.3 @@ -831,8 +831,8 @@ packages: '@textlint/types@15.8.0': resolution: {integrity: sha512-Anhc6y5736YIsvqae0U6k0YmB2M/QVHkEeOv2aydAn/WIkdI69dCOiDbe3/+RagS3qstFTSFWJzNRA2lUjv19w==} - '@toon-format/spec@4.1.2': - resolution: {integrity: sha512-e6IrY312qaBzY/O/wyxhICU28t/v85lrOTzEoFyMWQZBGTX0uxpUmlzaQsK8FL4S8n60MlU3QMIDQ27QQYiTjw==} + '@toon-format/spec@4.4.0': + resolution: {integrity: sha512-8KGd/+SttVZXDfICN8CtCAog1vW+Nq0o6mrSKBTBJYY8aDx1QLTcvpoJ05Uoc92Z3fNvtXllr8bibUZKVVOT0A==} '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -3952,7 +3952,7 @@ snapshots: dependencies: '@textlint/ast-node-types': 15.8.0 - '@toon-format/spec@4.1.2': {} + '@toon-format/spec@4.4.0': {} '@types/chai@5.2.3': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5a468a6..dfb8f7f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,6 @@ minimumReleaseAgeExcludePrune: true minimumReleaseAgeExclude: - - '@toon-format/spec@4.1.2' + - '@toon-format/spec@4.4.0' trustPolicy: no-downgrade trustPolicyExclude: diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index ec9d787..5c5f003 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -15,7 +15,6 @@ ], "repository": { "comment-line": { - "comment": "§5.1: comment lines", "match": "^ *(#).*$", "name": "comment.line.number-sign.toon", "captures": { @@ -25,16 +24,16 @@ } }, "table": { - "comment": "§9.3, §9.5, §10: a header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", + "comment": "A header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", "patterns": [ { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{)", + "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" } }, - "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))", + "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))", "patterns": [ { "include": "#comment-line" @@ -43,7 +42,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^\\s:\"\\[\\]{},])?)) *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^ :\"\\[\\]{},])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -71,13 +70,13 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{)", + "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" } }, - "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))", + "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))", "patterns": [ { "include": "#comment-line" @@ -86,7 +85,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^\\s:\"\\[\\]{}|])?)) *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^ :\"\\[\\]{}|])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -114,13 +113,13 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{)", + "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" } }, - "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))", + "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))", "patterns": [ { "include": "#comment-line" @@ -129,7 +128,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^\\s:\"\\[\\]{}])?)) *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^ \\t:\"\\[\\]{}])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -159,7 +158,7 @@ ] }, "line": { - "begin": "^[ \\t]*(?=\\S)", + "begin": "^[ \\t]*(?=[^ \\t\\n])", "end": "$", "patterns": [ { @@ -171,8 +170,7 @@ ] }, "list-item": { - "comment": "§9.4, §10: list items", - "begin": "\\G(-)(?: |$)", + "begin": "\\G(-)(?: +|$)", "beginCaptures": { "1": { "name": "punctuation.definition.list.begin.toon" @@ -199,8 +197,7 @@ ] }, "header": { - "comment": "§6: array headers, including keyed headers; inline values split by the bracket's delimiter", - "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=[{:])", + "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=:|\\{.*\\}:)", "beginCaptures": { "1": { "name": "support.type.property-name.toon", @@ -282,7 +279,6 @@ ] }, "field-list": { - "comment": "§6, §9.3: field lists with nested field groups", "begin": "\\{", "beginCaptures": { "0": { @@ -323,8 +319,7 @@ ] }, "key-value": { - "comment": "§5.2, §7.4: keys", - "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))[ ]*(:)(.*)$", + "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ :\"\\[\\]{}])?))?[ ]*(:)(.*)$", "captures": { "1": { "name": "support.type.property-name.toon", @@ -350,7 +345,7 @@ } }, "value": { - "comment": "§4, §11.2: a field value, list-item primitive, or root primitive is one token, delimiters included", + "comment": "A field value, list-item primitive, or root primitive is one token, delimiters included", "patterns": [ { "include": "#string-quoted" @@ -365,7 +360,7 @@ ] }, "cells-comma": { - "comment": "§9.1, §9.3, §11.2: inline array values and row cells, split by the active delimiter only; `[]` is a string here", + "comment": "Inline array values and row cells, split by the active delimiter only; `[]` is a string here", "patterns": [ { "include": "#string-quoted" @@ -435,7 +430,7 @@ ] }, "primitive": { - "comment": "§4: an unquoted token is a literal or a number only as a whole", + "comment": "An unquoted token is a literal or a number only as a whole", "patterns": [ { "match": "(?:true|false|null)(?= *$)", @@ -452,7 +447,6 @@ ] }, "string-quoted": { - "comment": "§7.1: quoted strings; an unterminated one is an error", "patterns": [ { "match": "\"(?:[^\"\\\\]|\\\\.)*\\\\?$", @@ -481,7 +475,6 @@ ] }, "string-escapes": { - "comment": "§7.1: escapes in quoted strings, keys, and field names", "patterns": [ { "match": "\\\\(?:[\\\\\"nrt]|u[0-9A-Fa-f]{4})", diff --git a/test/grammar.test.ts b/test/grammar.test.ts index b0a28b2..d540f17 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -39,35 +39,36 @@ function scopeOf(source: string, text: string): string | undefined { describe('grammar', () => { it.each([ - // §5.1 comment lines ['name: "a # b"', 'a # b', 'string.quoted.double.toon'], ['note: x #tag', 'x #tag', 'string.unquoted.toon'], - // §6 headers ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'], [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], - // §9.5 keyed tabular form + ['- key[1]: x', 'key', 'support.type.property-name.toon'], + ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], + // Whitespace is SP and HTAB only, so an NBSP or a trailing tab is part of the key + ['n [1]: y', 'n ', 'support.type.property-name.toon'], + ['a\t: 1', 'a\t', 'support.type.property-name.toon'], ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'], - // §5.2 key-value lines, §7.4 unquoted key tokens ['a:b[2]: x', 'a', 'support.type.property-name.toon'], ['foo-bar: 1', 'foo-bar', 'support.type.property-name.toon'], - // §12 the value token is trimmed, so the space after the colon is optional + [': 1', ':', 'punctuation.separator.key-value.toon'], + ['m[1:]{v}:\n : 4', ':', 'punctuation.separator.key-value.toon'], + // The value token is trimmed, so the space after the colon is optional ['a:30', '30', 'constant.numeric.toon'], ['a:true', 'true', 'constant.language.toon'], - // §4 number grammar ['n[4]: 1e5,-0,.5,05', '1e5', 'constant.numeric.toon'], ['n[4]: 1e5,-0,.5,05', '-0', 'constant.numeric.toon'], ['n[4]: 1e5,-0,.5,05', '.5', 'string.unquoted.toon'], ['n[4]: 1e5,-0,.5,05', '05', 'string.unquoted.toon'], ['n: +1', '+1', 'string.unquoted.toon'], - // §11.2 only the active delimiter splits, and only inline arrays and rows + // Only the active delimiter splits, and only inline arrays and rows ['n: a,b|c', 'a,b|c', 'string.unquoted.toon'], ['x[2|]: a,b|c', 'a,b', 'string.unquoted.toon'], ['- t[1|]{a|b}:\n x,y|z', 'x,y', 'string.unquoted.toon'], ['- t[1|]{a|b}:\n x|y\n k: a|b', 'a|b', 'string.unquoted.toon'], - // §9.3 `[]` is a string inside inline arrays and rows + // `[]` is a string inside inline arrays and rows ['tags[1]: []', '[]', 'string.unquoted.toon'], ['- []', '[]', 'constant.language.empty-array.toon'], - // §7.1 escapes ['k: "\\u00e9"', '\\u00e9', 'constant.character.escape.toon'], ['k: "\\/"', '\\/', 'invalid.illegal.unrecognized-string-escape.toon'], ['"a\\/b": 1', '\\/', 'invalid.illegal.unrecognized-string-escape.toon'], @@ -79,7 +80,7 @@ describe('grammar', () => { }) describe('spec decode fixtures', () => { - it.each(fixtureFiles)('%s: comment scope matches §5.1 exactly', async (file) => { + it.each(fixtureFiles)('%s: comment scope matches the comment lines exactly', async (file) => { const { tests } = JSON.parse(await readFile(new URL(file, fixtureDir), 'utf-8')) as { tests: { input: string }[] } for (const { input } of tests) { const lines = input.split('\n')