fix: align the grammar with TOON spec v4.4

README.md

@@ -1,7 +1,7 @@
 # TOON for Visual Studio Code
 
 [![Visual Studio Marketplace](https://img.shields.io/visual-studio-marketplace/v/toon-format.toon)](https://marketplace.visualstudio.com/items?itemName=toon-format.toon)
-[![SPEC v4.1](https://img.shields.io/badge/spec-v4.1-lightgrey)](https://github.com/toon-format/spec/blob/main/SPEC.md)
+[![SPEC v4.4](https://img.shields.io/badge/spec-v4.4-lightgrey)](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
 
 Highlights [TOON (Token-Oriented Object Notation)](https://github.com/toon-format/toon) in `.toon` files. TOON is a compact, indentation-based encoding of the JSON data model for LLM input.
@@ -34,7 +34,7 @@ Folding follows indentation, and pressing Enter after a line ending in `:` inden
 
 ## Specification
 
-Targets [TOON spec v4.1](https://github.com/toon-format/spec/blob/main/SPEC.md).
+Targets [TOON spec v4.4](https://github.com/toon-format/spec/blob/v4.4.0/SPEC.md).
 
 ## Resources
 

package.json

@@ -74,7 +74,7 @@
   },
   "devDependencies": {
     "@antfu/eslint-config": "^9.5.1",
-    "@toon-format/spec": "^4.1.2",
+    "@toon-format/spec": "^4.4.0",
     "@types/node": "^26.6.3",
     "@vscode/vsce": "^4.0.0",
     "bumpp": "^12.3.0",

pnpm-workspace.yaml

@@ -1,6 +1,6 @@
 minimumReleaseAgeExcludePrune: true
 minimumReleaseAgeExclude:
-  - '@toon-format/spec@4.1.2'
+  - '@toon-format/spec@4.4.0'
 
 trustPolicy: no-downgrade
 trustPolicyExclude:

syntaxes/toon.tmLanguage.json

@@ -15,7 +15,6 @@
   ],
   "repository": {
     "comment-line": {
-      "comment": "§5.1: comment lines",
       "match": "^ *(#).*$",
       "name": "comment.line.number-sign.toon",
       "captures": {
@@ -25,16 +24,16 @@
       }
     },
     "table": {
-      "comment": "§9.3, §9.5, §10: a header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields",
+      "comment": "A header with a field list and its rows, split by the header's delimiter. Rows sit deeper than the header, and under a list-item header deeper than the item's fields",
       "patterns": [
         {
-          "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{)",
+          "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\]\\{.*\\}:)",
           "beginCaptures": {
             "2": {
               "name": "punctuation.definition.list.begin.toon"
             }
           },
-          "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))",
+          "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))",
           "patterns": [
             {
               "include": "#comment-line"
@@ -43,7 +42,7 @@
               "include": "#header"
             },
             {
-              "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^\\s:\"\\[\\]{},])?)) *(:))?(.*)",
+              "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{},](?:[^:\"\\[\\]{},]*[^ :\"\\[\\]{},])?))? *(:))?(.*)",
               "captures": {
                 "1": {
                   "name": "support.type.property-name.toon",
@@ -71,13 +70,13 @@
           ]
         },
         {
-          "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{)",
+          "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\|\\]\\{.*\\}:)",
           "beginCaptures": {
             "2": {
               "name": "punctuation.definition.list.begin.toon"
             }
           },
-          "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))",
+          "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))",
           "patterns": [
             {
               "include": "#comment-line"
@@ -86,7 +85,7 @@
               "include": "#header"
             },
             {
-              "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^\\s:\"\\[\\]{}|])?)) *(:))?(.*)",
+              "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}|](?:[^:\"\\[\\]{}|]*[^ :\"\\[\\]{}|])?))? *(:))?(.*)",
               "captures": {
                 "1": {
                   "name": "support.type.property-name.toon",
@@ -114,13 +113,13 @@
           ]
         },
         {
-          "begin": "^( *)(?:(-)( ))?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{)",
+          "begin": "^( *)(?:(-)( ) *)?(?=(?:\"(?:[^\"\\\\]|\\\\.)*\"|[^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?)?\\[(?:0|[1-9][0-9]*):?\\t\\]\\{.*\\}:)",
           "beginCaptures": {
             "2": {
               "name": "punctuation.definition.list.begin.toon"
             }
           },
-          "while": "^(?=\\1\\3\\3 +\\S| *(?:#|$))",
+          "while": "^(?=\\1\\3\\3 +[^ \\t\\n]| *(?:#|$))",
           "patterns": [
             {
               "include": "#comment-line"
@@ -129,7 +128,7 @@
               "include": "#header"
             },
             {
-              "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^\\s:\"\\[\\]{}])?)) *(:))?(.*)",
+              "match": "^ +(?:(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}\\t]*[^ \\t:\"\\[\\]{}])?))? *(:))?(.*)",
               "captures": {
                 "1": {
                   "name": "support.type.property-name.toon",
@@ -159,7 +158,7 @@
       ]
     },
     "line": {
-      "begin": "^[ \\t]*(?=\\S)",
+      "begin": "^[ \\t]*(?=[^ \\t\\n])",
       "end": "$",
       "patterns": [
         {
@@ -171,8 +170,7 @@
       ]
     },
     "list-item": {
-      "comment": "§9.4, §10: list items",
-      "begin": "\\G(-)(?: |$)",
+      "begin": "\\G(-)(?: +|$)",
       "beginCaptures": {
         "1": {
           "name": "punctuation.definition.list.begin.toon"
@@ -199,8 +197,7 @@
       ]
     },
     "header": {
-      "comment": "§6: array headers, including keyed headers; inline values split by the bracket's delimiter",
-      "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=[{:])",
+      "begin": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ \\t:\"\\[\\]{}])?))?(\\[)(0|[1-9][0-9]*)(:)?(\\t|\\|)?(\\])(?=:|\\{.*\\}:)",
       "beginCaptures": {
         "1": {
           "name": "support.type.property-name.toon",
@@ -282,7 +279,6 @@
       ]
     },
     "field-list": {
-      "comment": "§6, §9.3: field lists with nested field groups",
       "begin": "\\{",
       "beginCaptures": {
         "0": {
@@ -323,8 +319,7 @@
       ]
     },
     "key-value": {
-      "comment": "§5.2, §7.4: keys",
-      "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^\\s:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^\\s:\"\\[\\]{}])?))[ ]*(:)(.*)$",
+      "match": "\\G(?:(\"(?:[^\"\\\\]|\\\\.)*\")|([^ \\t:\"\\[\\]{}](?:[^:\"\\[\\]{}]*[^ :\"\\[\\]{}])?))?[ ]*(:)(.*)$",
       "captures": {
         "1": {
           "name": "support.type.property-name.toon",
@@ -350,7 +345,7 @@
       }
     },
     "value": {
-      "comment": "§4, §11.2: a field value, list-item primitive, or root primitive is one token, delimiters included",
+      "comment": "A field value, list-item primitive, or root primitive is one token, delimiters included",
       "patterns": [
         {
           "include": "#string-quoted"
@@ -365,7 +360,7 @@
       ]
     },
     "cells-comma": {
-      "comment": "§9.1, §9.3, §11.2: inline array values and row cells, split by the active delimiter only; `[]` is a string here",
+      "comment": "Inline array values and row cells, split by the active delimiter only; `[]` is a string here",
       "patterns": [
         {
           "include": "#string-quoted"
@@ -435,7 +430,7 @@
       ]
     },
     "primitive": {
-      "comment": "§4: an unquoted token is a literal or a number only as a whole",
+      "comment": "An unquoted token is a literal or a number only as a whole",
       "patterns": [
         {
           "match": "(?:true|false|null)(?= *$)",
@@ -452,7 +447,6 @@
       ]
     },
     "string-quoted": {
-      "comment": "§7.1: quoted strings; an unterminated one is an error",
       "patterns": [
         {
           "match": "\"(?:[^\"\\\\]|\\\\.)*\\\\?$",
@@ -481,7 +475,6 @@
       ]
     },
     "string-escapes": {
-      "comment": "§7.1: escapes in quoted strings, keys, and field names",
       "patterns": [
         {
           "match": "\\\\(?:[\\\\\"nrt]|u[0-9A-Fa-f]{4})",

test/grammar.test.ts

@@ -39,35 +39,36 @@ function scopeOf(source: string, text: string): string | undefined {
 
 describe('grammar', () => {
   it.each([
-    // §5.1 comment lines
     ['name: "a # b"', 'a # b', 'string.quoted.double.toon'],
     ['note: x #tag', 'x #tag', 'string.unquoted.toon'],
-    // §6 headers
     ['"a:b"[2]: 1,2', '"a:b"', 'support.type.property-name.toon'],
     ['  - key[2]{a,b}:', 'key', 'support.type.property-name.toon'],
-    // §9.5 keyed tabular form
+    ['-  key[1]: x', 'key', 'support.type.property-name.toon'],
+    ['a[2:]{x}', 'a[2:]{x}', 'string.unquoted.toon'],
+    // Whitespace is SP and HTAB only, so an NBSP or a trailing tab is part of the key
+    ['n [1]: y', 'n ', 'support.type.property-name.toon'],
+    ['a\t: 1', 'a\t', 'support.type.property-name.toon'],
     ['[2:|]{a|b}:', 'b', 'support.type.property-name.field.toon'],
-    // §5.2 key-value lines, §7.4 unquoted key tokens
     ['a:b[2]: x', 'a', 'support.type.property-name.toon'],
     ['foo-bar: 1', 'foo-bar', 'support.type.property-name.toon'],
-    // §12 the value token is trimmed, so the space after the colon is optional
+    [': 1', ':', 'punctuation.separator.key-value.toon'],
+    ['m[1:]{v}:\n  : 4', ':', 'punctuation.separator.key-value.toon'],
+    // The value token is trimmed, so the space after the colon is optional
     ['a:30', '30', 'constant.numeric.toon'],
     ['a:true', 'true', 'constant.language.toon'],
-    // §4 number grammar
     ['n[4]: 1e5,-0,.5,05', '1e5', 'constant.numeric.toon'],
     ['n[4]: 1e5,-0,.5,05', '-0', 'constant.numeric.toon'],
     ['n[4]: 1e5,-0,.5,05', '.5', 'string.unquoted.toon'],
     ['n[4]: 1e5,-0,.5,05', '05', 'string.unquoted.toon'],
     ['n: +1', '+1', 'string.unquoted.toon'],
-    // §11.2 only the active delimiter splits, and only inline arrays and rows
+    // Only the active delimiter splits, and only inline arrays and rows
     ['n: a,b|c', 'a,b|c', 'string.unquoted.toon'],
     ['x[2|]: a,b|c', 'a,b', 'string.unquoted.toon'],
     ['- t[1|]{a|b}:\n    x,y|z', 'x,y', 'string.unquoted.toon'],
     ['- t[1|]{a|b}:\n    x|y\n  k: a|b', 'a|b', 'string.unquoted.toon'],
-    // §9.3 `[]` is a string inside inline arrays and rows
+    // `[]` is a string inside inline arrays and rows
     ['tags[1]: []', '[]', 'string.unquoted.toon'],
     ['- []', '[]', 'constant.language.empty-array.toon'],
-    // §7.1 escapes
     ['k: "\\u00e9"', '\\u00e9', 'constant.character.escape.toon'],
     ['k: "\\/"', '\\/', 'invalid.illegal.unrecognized-string-escape.toon'],
     ['"a\\/b": 1', '\\/', 'invalid.illegal.unrecognized-string-escape.toon'],
@@ -79,7 +80,7 @@ describe('grammar', () => {
 })
 
 describe('spec decode fixtures', () => {
-  it.each(fixtureFiles)('%s: comment scope matches §5.1 exactly', async (file) => {
+  it.each(fixtureFiles)('%s: comment scope matches the comment lines exactly', async (file) => {
     const { tests } = JSON.parse(await readFile(new URL(file, fixtureDir), 'utf-8')) as { tests: { input: string }[] }
     for (const { input } of tests) {
       const lines = input.split('\n')