From 8487bc5cd19ce6fd8a806a178f14af7f6e8ef309 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:20:15 +0200 Subject: [PATCH 01/12] chore: update `@toon-format/spec` to v4.3.0 --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- pnpm-workspace.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 3ac4cea..230d679 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ }, "devDependencies": { "@antfu/eslint-config": "^9.5.1", - "@toon-format/spec": "^4.1.2", + "@toon-format/spec": "^4.3.0", "@types/node": "^26.6.3", "@vscode/vsce": "^4.0.0", "bumpp": "^12.3.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 085eb06..fcdb19b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^9.5.1 version: 9.5.1(@typescript-eslint/typescript-estree@8.71.0(supports-color@7.2.0)(typescript@6.0.3))(@typescript-eslint/utils@8.71.0(eslint@10.11.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(@vue/compiler-sfc@3.5.25)(eslint@10.11.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(ts-declaration-location@1.0.7(typescript@6.0.3))(typescript@6.0.3)(vitest@5.0.3(@types/node@26.6.3)(vite@8.3.1(@types/node@26.6.3)(jiti@2.7.0)(yaml@2.9.1))) '@toon-format/spec': - specifier: ^4.1.2 - version: 4.1.2 + specifier: ^4.3.0 + version: 4.3.0 '@types/node': specifier: ^26.6.3 version: 26.6.3 @@ -831,8 +831,8 @@ packages: '@textlint/types@15.8.0': resolution: {integrity: sha512-Anhc6y5736YIsvqae0U6k0YmB2M/QVHkEeOv2aydAn/WIkdI69dCOiDbe3/+RagS3qstFTSFWJzNRA2lUjv19w==} - '@toon-format/spec@4.1.2': - resolution: {integrity: sha512-e6IrY312qaBzY/O/wyxhICU28t/v85lrOTzEoFyMWQZBGTX0uxpUmlzaQsK8FL4S8n60MlU3QMIDQ27QQYiTjw==} + '@toon-format/spec@4.3.0': + resolution: {integrity: sha512-RN2tNyWer+geNO33HhsKCleH+YsMggBaH+2tSIY43sJJAivCzguce5jcAthBd+RTTDr8qM2UvhElPDIeuVt2Kw==} '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -3952,7 +3952,7 @@ snapshots: dependencies: '@textlint/ast-node-types': 15.8.0 - '@toon-format/spec@4.1.2': {} + '@toon-format/spec@4.3.0': {} '@types/chai@5.2.3': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5a468a6..558166a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,6 @@ minimumReleaseAgeExcludePrune: true minimumReleaseAgeExclude: - - '@toon-format/spec@4.1.2' + - '@toon-format/spec@4.3.0' trustPolicy: no-downgrade trustPolicyExclude: From ed0b9e1664837bc4203c446a7038b9334ce08bba Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:20:16 +0200 Subject: [PATCH 02/12] fix: scope the empty key of a key-value line or entry row --- syntaxes/toon.tmLanguage.json | 8 ++++---- test/grammar.test.ts | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index ec9d787..b85bda3 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -43,7 +43,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^\\s:\"\\[\\]{},])?)) *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^\\s:\"\\[\\]{},])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -86,7 +86,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^\\s:\"\\[\\]{}|])?)) *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^\\s:\"\\[\\]{}|])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -129,7 +129,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^\\s:\"\\[\\]{}])?)) *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^\\s:\"\\[\\]{}])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -324,7 +324,7 @@ }, "key-value": { "comment": "§5.2, §7.4: keys", - "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))[ ]*(:)(.*)$", + "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?[ ]*(:)(.*)$", "captures": { "1": { "name": "support.type.property-name.toon", diff --git a/test/grammar.test.ts b/test/grammar.test.ts index b0a28b2..2a338f1 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -50,6 +50,8 @@ describe('grammar', () => { // §5.2 key-value lines, §7.4 unquoted key tokens ['a:b[2]: x', 'a', 'support.type.property-name.toon'], ['foo-bar: 1', 'foo-bar', 'support.type.property-name.toon'], + [': 1', ':', 'punctuation.separator.key-value.toon'], + ['m[1:]{v}:\n : 4', ':', 'punctuation.separator.key-value.toon'], // §12 the value token is trimmed, so the space after the colon is optional ['a:30', '30', 'constant.numeric.toon'], ['a:true', 'true', 'constant.language.toon'], From 52c8f2a7117e39e65cd41e5abb22caf681149b06 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:20:16 +0200 Subject: [PATCH 03/12] docs: declare spec v4.3 as the target --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3c3e8e2..603887a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # TOON for Visual Studio Code [![Visual Studio Marketplace](https://img.shields.io/visual-studio-marketplace/v/toon-format.toon)](https://marketplace.visualstudio.com/items?itemName=toon-format.toon) -[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-lightgrey)](https://github.com/toon-format/spec/blob/main/SPEC.md) +[![SPEC v4.3](https://img.shields.io/badge/spec-v4.3-lightgrey)](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) Highlights [TOON (Token-Oriented Object Notation)](https://github.com/toon-format/toon) in `.toon` files. TOON is a compact, indentation-based encoding of the JSON data model for LLM input. @@ -34,7 +34,7 @@ Folding follows indentation, and pressing Enter after a line ending in `:` inden ## Specification -Targets [TOON spec v4.1](https://github.com/toon-format/spec/blob/main/SPEC.md). +Targets [TOON spec v4.3](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md). ## Resources From 7e79587e044f27d1df8cdd916b74bbb92d2b596c Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:28:13 +0200 Subject: [PATCH 04/12] fix: require the colon after a header's field list --- syntaxes/toon.tmLanguage.json | 8 ++++---- test/grammar.test.ts | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index b85bda3..31a9b33 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -28,7 +28,7 @@ "comment": "§9.3, §9.5, §10: a header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", "patterns": [ { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{)", + "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" @@ -71,7 +71,7 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{)", + "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" @@ -114,7 +114,7 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{)", + "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" @@ -200,7 +200,7 @@ }, "header": { "comment": "§6: array headers, including keyed headers; inline values split by the bracket's delimiter", - "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=[{:])", + "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=:|\\{.*\\}:)", "beginCaptures": { "1": { "name": "support.type.property-name.toon", diff --git a/test/grammar.test.ts b/test/grammar.test.ts index 2a338f1..50a257b 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -45,6 +45,7 @@ describe('grammar', () => { // §6 headers ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'], [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], + ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], // §9.5 keyed tabular form ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'], // §5.2 key-value lines, §7.4 unquoted key tokens From 9f5cbc87283da623b94a43d389092feaf899f731 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Tue, 6 Oct 2026 16:28:46 +0200 Subject: [PATCH 05/12] chore: drop spec section references from grammar comments --- syntaxes/toon.tmLanguage.json | 15 ++++----------- test/grammar.test.ts | 14 ++++---------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index 31a9b33..4579baf 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -15,7 +15,6 @@ ], "repository": { "comment-line": { - "comment": "§5.1: comment lines", "match": "^ *(#).*$", "name": "comment.line.number-sign.toon", "captures": { @@ -25,7 +24,7 @@ } }, "table": { - "comment": "§9.3, §9.5, §10: a header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", + "comment": "A header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", "patterns": [ { "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", @@ -171,7 +170,6 @@ ] }, "list-item": { - "comment": "§9.4, §10: list items", "begin": "\\G(-)(?: |$)", "beginCaptures": { "1": { @@ -199,7 +197,6 @@ ] }, "header": { - "comment": "§6: array headers, including keyed headers; inline values split by the bracket's delimiter", "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=:|\\{.*\\}:)", "beginCaptures": { "1": { @@ -282,7 +279,6 @@ ] }, "field-list": { - "comment": "§6, §9.3: field lists with nested field groups", "begin": "\\{", "beginCaptures": { "0": { @@ -323,7 +319,6 @@ ] }, "key-value": { - "comment": "§5.2, §7.4: keys", "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?[ ]*(:)(.*)$", "captures": { "1": { @@ -350,7 +345,7 @@ } }, "value": { - "comment": "§4, §11.2: a field value, list-item primitive, or root primitive is one token, delimiters included", + "comment": "A field value, list-item primitive, or root primitive is one token, delimiters included", "patterns": [ { "include": "#string-quoted" @@ -365,7 +360,7 @@ ] }, "cells-comma": { - "comment": "§9.1, §9.3, §11.2: inline array values and row cells, split by the active delimiter only; `[]` is a string here", + "comment": "Inline array values and row cells, split by the active delimiter only; `[]` is a string here", "patterns": [ { "include": "#string-quoted" @@ -435,7 +430,7 @@ ] }, "primitive": { - "comment": "§4: an unquoted token is a literal or a number only as a whole", + "comment": "An unquoted token is a literal or a number only as a whole", "patterns": [ { "match": "(?:true|false|null)(?= *$)", @@ -452,7 +447,6 @@ ] }, "string-quoted": { - "comment": "§7.1: quoted strings; an unterminated one is an error", "patterns": [ { "match": "\"(?:[^\"\\\\]|\\\\.)*\\\\?$", @@ -481,7 +475,6 @@ ] }, "string-escapes": { - "comment": "§7.1: escapes in quoted strings, keys, and field names", "patterns": [ { "match": "\\\\(?:[\\\\\"nrt]|u[0-9A-Fa-f]{4})", diff --git a/test/grammar.test.ts b/test/grammar.test.ts index 50a257b..354f03e 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -39,38 +39,32 @@ function scopeOf(source: string, text: string): string | undefined { describe('grammar', () => { it.each([ - // §5.1 comment lines ['name: "a # b"', 'a # b', 'string.quoted.double.toon'], ['note: x #tag', 'x #tag', 'string.unquoted.toon'], - // §6 headers ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'], [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], - // §9.5 keyed tabular form ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'], - // §5.2 key-value lines, §7.4 unquoted key tokens ['a:b[2]: x', 'a', 'support.type.property-name.toon'], ['foo-bar: 1', 'foo-bar', 'support.type.property-name.toon'], [': 1', ':', 'punctuation.separator.key-value.toon'], ['m[1:]{v}:\n : 4', ':', 'punctuation.separator.key-value.toon'], - // §12 the value token is trimmed, so the space after the colon is optional + // The value token is trimmed, so the space after the colon is optional ['a:30', '30', 'constant.numeric.toon'], ['a:true', 'true', 'constant.language.toon'], - // §4 number grammar ['n[4]: 1e5,-0,.5,05', '1e5', 'constant.numeric.toon'], ['n[4]: 1e5,-0,.5,05', '-0', 'constant.numeric.toon'], ['n[4]: 1e5,-0,.5,05', '.5', 'string.unquoted.toon'], ['n[4]: 1e5,-0,.5,05', '05', 'string.unquoted.toon'], ['n: +1', '+1', 'string.unquoted.toon'], - // §11.2 only the active delimiter splits, and only inline arrays and rows + // Only the active delimiter splits, and only inline arrays and rows ['n: a,b|c', 'a,b|c', 'string.unquoted.toon'], ['x[2|]: a,b|c', 'a,b', 'string.unquoted.toon'], ['- t[1|]{a|b}:\n x,y|z', 'x,y', 'string.unquoted.toon'], ['- t[1|]{a|b}:\n x|y\n k: a|b', 'a|b', 'string.unquoted.toon'], - // §9.3 `[]` is a string inside inline arrays and rows + // `[]` is a string inside inline arrays and rows ['tags[1]: []', '[]', 'string.unquoted.toon'], ['- []', '[]', 'constant.language.empty-array.toon'], - // §7.1 escapes ['k: "\\u00e9"', '\\u00e9', 'constant.character.escape.toon'], ['k: "\\/"', '\\/', 'invalid.illegal.unrecognized-string-escape.toon'], ['"a\\/b": 1', '\\/', 'invalid.illegal.unrecognized-string-escape.toon'], @@ -82,7 +76,7 @@ describe('grammar', () => { }) describe('spec decode fixtures', () => { - it.each(fixtureFiles)('%s: comment scope matches §5.1 exactly', async (file) => { + it.each(fixtureFiles)('%s: comment scope matches the comment lines exactly', async (file) => { const { tests } = JSON.parse(await readFile(new URL(file, fixtureDir), 'utf-8')) as { tests: { input: string }[] } for (const { input } of tests) { const lines = input.split('\n') From c92bca63e7db5156fd208538ac66fa5984cb6066 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 09:50:22 +0200 Subject: [PATCH 06/12] fix: treat only SP and HTAB as whitespace --- syntaxes/toon.tmLanguage.json | 24 ++++++++++++------------ test/grammar.test.ts | 2 ++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index 4579baf..104b49b 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -27,13 +27,13 @@ "comment": "A header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", "patterns": [ { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", + "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" } }, - "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))", + "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))", "patterns": [ { "include": "#comment-line" @@ -42,7 +42,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^\\s:\"\\[\\]{},])?))? *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^ \\t:\"\\[\\]{},])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -70,13 +70,13 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)", + "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" } }, - "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))", + "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))", "patterns": [ { "include": "#comment-line" @@ -85,7 +85,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^\\s:\"\\[\\]{}|])?))? *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^ \\t:\"\\[\\]{}|])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -113,13 +113,13 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)", + "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" } }, - "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))", + "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))", "patterns": [ { "include": "#comment-line" @@ -128,7 +128,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^\\s:\"\\[\\]{}])?))? *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^ \\t:\"\\[\\]{}])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -158,7 +158,7 @@ ] }, "line": { - "begin": "^[ \\t]*(?=\\S)", + "begin": "^[ \\t]*(?=[^ \\t\\n])", "end": "$", "patterns": [ { @@ -197,7 +197,7 @@ ] }, "header": { - "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=:|\\{.*\\}:)", + "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=:|\\{.*\\}:)", "beginCaptures": { "1": { "name": "support.type.property-name.toon", @@ -319,7 +319,7 @@ ] }, "key-value": { - "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?[ ]*(:)(.*)$", + "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?))?[ ]*(:)(.*)$", "captures": { "1": { "name": "support.type.property-name.toon", diff --git a/test/grammar.test.ts b/test/grammar.test.ts index 354f03e..03a18ec 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -44,6 +44,8 @@ describe('grammar', () => { ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'], [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], + // Whitespace is SP and HTAB only, so an NBSP is part of the key + ['n [1]: y', 'n ', 'support.type.property-name.toon'], ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'], ['a:b[2]: x', 'a', 'support.type.property-name.toon'], ['foo-bar: 1', 'foo-bar', 'support.type.property-name.toon'], From 2dbd4798d126372b71c679fc74dda18892b0b90b Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 09:50:38 +0200 Subject: [PATCH 07/12] fix: accept several spaces after a list-item hyphen --- syntaxes/toon.tmLanguage.json | 8 ++++---- test/grammar.test.ts | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index 104b49b..45d4459 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -27,7 +27,7 @@ "comment": "A header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields", "patterns": [ { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", + "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" @@ -70,7 +70,7 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)", + "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" @@ -113,7 +113,7 @@ ] }, { - "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)", + "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)", "beginCaptures": { "2": { "name": "punctuation.definition.list.begin.toon" @@ -170,7 +170,7 @@ ] }, "list-item": { - "begin": "\\G(-)(?: |$)", + "begin": "\\G(-)(?: +|$)", "beginCaptures": { "1": { "name": "punctuation.definition.list.begin.toon" diff --git a/test/grammar.test.ts b/test/grammar.test.ts index 03a18ec..bee4895 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -43,6 +43,8 @@ describe('grammar', () => { ['note: x #tag', 'x #tag', 'string.unquoted.toon'], ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'], [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], + // A list-item hyphen may be followed by several spaces + ['- key[1]: x', 'key', 'support.type.property-name.toon'], ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], // Whitespace is SP and HTAB only, so an NBSP is part of the key ['n [1]: y', 'n ', 'support.type.property-name.toon'], From e6916615597039ad89791325da690a1b8e5f418f Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 10:51:21 +0200 Subject: [PATCH 08/12] test: drop a comment that restates its case --- test/grammar.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/grammar.test.ts b/test/grammar.test.ts index bee4895..36fc4fd 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -43,7 +43,6 @@ describe('grammar', () => { ['note: x #tag', 'x #tag', 'string.unquoted.toon'], ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'], [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], - // A list-item hyphen may be followed by several spaces ['- key[1]: x', 'key', 'support.type.property-name.toon'], ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], // Whitespace is SP and HTAB only, so an NBSP is part of the key From 5dbdbc00b2c921bdcb8f7c82891699d40f6738fa Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Wed, 7 Oct 2026 10:52:06 +0200 Subject: [PATCH 09/12] fix: keep a trailing tab in a key --- syntaxes/toon.tmLanguage.json | 6 +++--- test/grammar.test.ts | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/syntaxes/toon.tmLanguage.json b/syntaxes/toon.tmLanguage.json index 45d4459..5c5f003 100644 --- a/syntaxes/toon.tmLanguage.json +++ b/syntaxes/toon.tmLanguage.json @@ -42,7 +42,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^ \\t:\"\\[\\]{},])?))? *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^ :\"\\[\\]{},])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -85,7 +85,7 @@ "include": "#header" }, { - "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^ \\t:\"\\[\\]{}|])?))? *(:))?(.*)", + "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^ :\"\\[\\]{}|])?))? *(:))?(.*)", "captures": { "1": { "name": "support.type.property-name.toon", @@ -319,7 +319,7 @@ ] }, "key-value": { - "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?))?[ ]*(:)(.*)$", + "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ :\"\\[\\]{}])?))?[ ]*(:)(.*)$", "captures": { "1": { "name": "support.type.property-name.toon", diff --git a/test/grammar.test.ts b/test/grammar.test.ts index 36fc4fd..3b979b0 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -47,6 +47,7 @@ describe('grammar', () => { ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], // Whitespace is SP and HTAB only, so an NBSP is part of the key ['n [1]: y', 'n ', 'support.type.property-name.toon'], + ['a\t: 1', 'a\t', 'support.type.property-name.toon'], ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'], ['a:b[2]: x', 'a', 'support.type.property-name.toon'], ['foo-bar: 1', 'foo-bar', 'support.type.property-name.toon'], From 93f64b929d4a12d752eded5f0da24a3cd59f8f78 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Thu, 8 Oct 2026 17:53:00 +0200 Subject: [PATCH 10/12] chore: update `@toon-format/spec` to v4.4.0 --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- pnpm-workspace.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 230d679..e56d438 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ }, "devDependencies": { "@antfu/eslint-config": "^9.5.1", - "@toon-format/spec": "^4.3.0", + "@toon-format/spec": "^4.4.0", "@types/node": "^26.6.3", "@vscode/vsce": "^4.0.0", "bumpp": "^12.3.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fcdb19b..8c5eddc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^9.5.1 version: 9.5.1(@typescript-eslint/typescript-estree@8.71.0(supports-color@7.2.0)(typescript@6.0.3))(@typescript-eslint/utils@8.71.0(eslint@10.11.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(@vue/compiler-sfc@3.5.25)(eslint@10.11.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(ts-declaration-location@1.0.7(typescript@6.0.3))(typescript@6.0.3)(vitest@5.0.3(@types/node@26.6.3)(vite@8.3.1(@types/node@26.6.3)(jiti@2.7.0)(yaml@2.9.1))) '@toon-format/spec': - specifier: ^4.3.0 - version: 4.3.0 + specifier: ^4.4.0 + version: 4.4.0 '@types/node': specifier: ^26.6.3 version: 26.6.3 @@ -831,8 +831,8 @@ packages: '@textlint/types@15.8.0': resolution: {integrity: sha512-Anhc6y5736YIsvqae0U6k0YmB2M/QVHkEeOv2aydAn/WIkdI69dCOiDbe3/+RagS3qstFTSFWJzNRA2lUjv19w==} - '@toon-format/spec@4.3.0': - resolution: {integrity: sha512-RN2tNyWer+geNO33HhsKCleH+YsMggBaH+2tSIY43sJJAivCzguce5jcAthBd+RTTDr8qM2UvhElPDIeuVt2Kw==} + '@toon-format/spec@4.4.0': + resolution: {integrity: sha512-8KGd/+SttVZXDfICN8CtCAog1vW+Nq0o6mrSKBTBJYY8aDx1QLTcvpoJ05Uoc92Z3fNvtXllr8bibUZKVVOT0A==} '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -3952,7 +3952,7 @@ snapshots: dependencies: '@textlint/ast-node-types': 15.8.0 - '@toon-format/spec@4.3.0': {} + '@toon-format/spec@4.4.0': {} '@types/chai@5.2.3': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 558166a..dfb8f7f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,6 @@ minimumReleaseAgeExcludePrune: true minimumReleaseAgeExclude: - - '@toon-format/spec@4.3.0' + - '@toon-format/spec@4.4.0' trustPolicy: no-downgrade trustPolicyExclude: From d52adca5a1b645eaba8f06d4518e20048a6bcaea Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Thu, 8 Oct 2026 17:53:06 +0200 Subject: [PATCH 11/12] docs: declare spec v4.4 as the target --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 603887a..126adf2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # TOON for Visual Studio Code [![Visual Studio Marketplace](https://img.shields.io/visual-studio-marketplace/v/toon-format.toon)](https://marketplace.visualstudio.com/items?itemName=toon-format.toon) -[![SPEC v4.3](https://img.shields.io/badge/spec-v4.3-lightgrey)](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md) +[![SPEC v4.4](https://img.shields.io/badge/spec-v4.4-lightgrey)](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) Highlights [TOON (Token-Oriented Object Notation)](https://github.com/toon-format/toon) in `.toon` files. TOON is a compact, indentation-based encoding of the JSON data model for LLM input. @@ -34,7 +34,7 @@ Folding follows indentation, and pressing Enter after a line ending in `:` inden ## Specification -Targets [TOON spec v4.3](https://github.com/toon-format/spec/blob/v4.3.0/SPEC.md). +Targets [TOON spec v4.4](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md). ## Resources From 0b1afe2c4cc711c21293c9a77d141aa44819c3ea Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Thu, 8 Oct 2026 17:57:01 +0200 Subject: [PATCH 12/12] fixup! fix: keep a trailing tab in a key --- test/grammar.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/grammar.test.ts b/test/grammar.test.ts index 3b979b0..d540f17 100644 --- a/test/grammar.test.ts +++ b/test/grammar.test.ts @@ -45,7 +45,7 @@ describe('grammar', () => { [' - key[2]{a,b}:', 'key', 'support.type.property-name.toon'], ['- key[1]: x', 'key', 'support.type.property-name.toon'], ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'], - // Whitespace is SP and HTAB only, so an NBSP is part of the key + // Whitespace is SP and HTAB only, so an NBSP or a trailing tab is part of the key ['n [1]: y', 'n ', 'support.type.property-name.toon'], ['a\t: 1', 'a\t', 'support.type.property-name.toon'], ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'],