From adef08ded32413d123340fa0f4a1cd69e3c3bb30 Mon Sep 17 00:00:00 2001 From: John Ky Date: Wed, 11 Nov 2026 17:51:33 +1100 Subject: [PATCH 1/2] fix(yaml): read a tab after a closing quote or an alias as separation (#6937) The scanners for a quoted scalar and an alias stop just past their closing delimiter, so parse_document_line can be entered mid-line. count_indent took a tab at the cursor for indentation, and the tab-only-line check took the cursor for the start of a blank line, so `a: "b"`, `"a"`, `- 'x'`, `a: &x "b"` and `a: *x` were refused where yq reads them as separation. Both checks now apply only where the cursor is at the start of a line (at_line_start, which reject_tab_only_line_at_cursor also uses). A tab that starts a line and content after the tab stay refused, as in yq. --- changelog.d/6937.fixed.md | 1 + src/yaml/parser.rs | 24 ++++++++++++--- tests/yq_cli/i6937.rs | 64 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 changelog.d/6937.fixed.md create mode 100644 tests/yq_cli/i6937.rs diff --git a/changelog.d/6937.fixed.md b/changelog.d/6937.fixed.md new file mode 100644 index 000000000..f3ab24e98 --- /dev/null +++ b/changelog.d/6937.fixed.md @@ -0,0 +1 @@ +- **yq: a tab right after a quoted scalar or an alias in block context is read as separation** (#6937). `a: "b"⇥`, `"a"⇥`, `- 'x'⇥`, `a: &x "b"⇥` and `a: *x⇥` were a parse error ("tab character used for indentation"), where yq reads them: the scanners for a quoted scalar and an alias stop just past their closing delimiter, so the main loop was entered mid-line and took the tab at the cursor for indentation, or for a line of its own. A tab that starts a line, and content after the tab (`a: "b"⇥c`), are still refused, as in yq. Found by a mutation fuzz of the yaml-test-suite inputs against yq v4.53.3. diff --git a/src/yaml/parser.rs b/src/yaml/parser.rs index d7ff1ca60..14f629d0b 100644 --- a/src/yaml/parser.rs +++ b/src/yaml/parser.rs @@ -3030,6 +3030,15 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { } } + /// Whether the cursor is at the start of a line: the start of the input, or just past a break. + /// + /// The scanners for a quoted scalar, a flow collection and an alias stop just past their + /// closing delimiter, so the main loop can be entered mid-line (#6937). + #[inline] + fn at_line_start(&self) -> bool { + self.pos == 0 || Self::is_break(self.input[self.pos - 1]) + } + /// Count leading spaces (indentation) at start of a line. fn count_indent(&self) -> Result { // Use SIMD-accelerated space counting @@ -3041,7 +3050,10 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { // Tab after spaces - check context // If we haven't seen any spaces and hit a tab at start of line, // that's tab indentation (error). But tab after spaces is content. - if count == 0 { + // #6937: only at the start of a line. The quoted-scalar and alias scanners stop just + // past their delimiter, and a tab there (`a: "b"⇥`) is separation before the line break + // or a comment, which yq reads. + if count == 0 && self.at_line_start() { return Err(YamlError::TabIndentation { line: self.current_line(), offset: next_pos, @@ -3145,8 +3157,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { /// past an empty value or key and would otherwise take the line as its blank prefix (#6295: `a:` / /// ` ⇥` is refused by yq, and was read as `{"a": null}`; `?` / ` ⇥` / ` a` as `{"a": null}` too). fn reject_tab_only_line_at_cursor(&self) -> Result<(), YamlError> { - let at_line_start = self.pos == 0 || Self::is_break(self.input[self.pos - 1]); - if at_line_start && self.tab_only_line_is_refused(self.pos) { + if self.at_line_start() && self.tab_only_line_is_refused(self.pos) { return Err(YamlError::TabIndentation { line: self.current_line(), offset: self.pos, @@ -10211,6 +10222,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { // Skip to content let line_start = self.pos; + let entered_at_line_start = self.at_line_start(); self.advance_by(indent); // A tab *after* the leading spaces is indentation — and so illegal — only when @@ -10225,7 +10237,11 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { } // The cursor is past the line's spaces: only a line whose next byte is the tab can be the one. - if self.peek() == Some(b'\t') && self.tab_only_line_is_refused(line_start) { + // A tab after a closing quote or an alias is no line of its own (#6937). + if entered_at_line_start + && self.peek() == Some(b'\t') + && self.tab_only_line_is_refused(line_start) + { return Err(YamlError::TabIndentation { line: self.current_line(), offset: self.pos, diff --git a/tests/yq_cli/i6937.rs b/tests/yq_cli/i6937.rs new file mode 100644 index 000000000..c6b470804 --- /dev/null +++ b/tests/yq_cli/i6937.rs @@ -0,0 +1,64 @@ +//! #6937: a tab directly after the closing quote of a quoted scalar, or after an alias, in block context is +//! separation before the line break or a comment, as it is after a plain scalar or a flow collection. The +//! scanners for those stop just past their closing delimiter, so the main loop was entered mid-line and read +//! the tab at the cursor as indentation. A tab that starts a line is still refused, and so is content after it. +//! +//! Every row is captured from yq v4.53.3 (`-o=json -I=0 .`). + +use anyhow::Result; + +use crate::run_yq_stdin; + +/// Accepted by yq: `(document, expected output)`. +const ACCEPTED: &[(&str, &str)] = &[ + ("\"a\"\t\n", "\"a\"\n"), + ("'a'\t\n", "\"a\"\n"), + ("a: \"b\"\t\n", "{\"a\":\"b\"}\n"), + ("a: \"b\"\t# c\n", "{\"a\":\"b\"}\n"), + ("a: \"b\"\t \n", "{\"a\":\"b\"}\n"), + ("a: \"b\" \t\n", "{\"a\":\"b\"}\n"), + ("- \"x\"\t\n- y\n", "[\"x\",\"y\"]\n"), + ("? \"a\"\t\n: 1\n", "{\"a\":1}\n"), + ("a: &x \"b\"\t\n", "{\"a\":\"b\"}\n"), + ("x: &x 1\na: *x\t\n", "{\"x\":1,\"a\":1}\n"), + ("a: !!str \"b\"\t\n", "{\"a\":\"b\"}\n"), + ("a: \"b\n c\"\t\n", "{\"a\":\"b c\"}\n"), + ("\"a\"\t", "\"a\"\n"), + ("\"a\"\t\t\n", "\"a\"\n"), + ("a: \"b\"\t\nc: 1\n", "{\"a\":\"b\",\"c\":1}\n"), + ("a:\n - \"b\"\t\n - c\n", "{\"a\":[\"b\",\"c\"]}\n"), + ("a: b\t\n", "{\"a\":\"b\"}\n"), + ("[1]\t\n", "[1]\n"), + ("a: {x: 1}\t\n", "{\"a\":{\"x\":1}}\n"), +]; + +/// Refused by yq (a tab that starts a line, or content after the tab). +const REFUSED: &[&str] = &[ + "\"a\"\n\t\n", + "a: \"b\"\n\t\n", + "\t\"a\"\n", + "a:\n\t\"b\"\n", + "a: \"b\"\tc\n", + "\"a\"\tb\n", + "- \"a\"\t- b\n", + "\"a\"\n \t\n", +]; + +#[test] +fn a_tab_after_a_closing_quote_or_an_alias_is_separation_6937() -> Result<()> { + for (doc, want) in ACCEPTED { + let (out, code) = run_yq_stdin(".", doc, &["-o=json", "-I=0"])?; + assert_eq!(code, 0, "{doc:?}"); + assert_eq!(out, *want, "{doc:?}"); + } + Ok(()) +} + +#[test] +fn a_tab_that_starts_a_line_or_precedes_content_is_still_refused_6937() -> Result<()> { + for doc in REFUSED { + let (_, code) = run_yq_stdin(".", doc, &["-o=json", "-I=0"])?; + assert_ne!(code, 0, "{doc:?}"); + } + Ok(()) +} From c5a054e562df173fdefc3e4141b7b689a8461c0a Mon Sep 17 00:00:00 2001 From: John Ky Date: Wed, 11 Nov 2026 17:58:45 +1100 Subject: [PATCH 2/2] test(yaml): CRLF, document-marker and mid-line rows for #6937 review Also corrects the count_indent / tab_indents_block_structure comments: Ok(0) no longer implies a tab is absent at the cursor. --- src/yaml/parser.rs | 7 ++++--- tests/yq_cli/i6937.rs | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/yaml/parser.rs b/src/yaml/parser.rs index 14f629d0b..9e58374e7 100644 --- a/src/yaml/parser.rs +++ b/src/yaml/parser.rs @@ -3059,7 +3059,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { offset: next_pos, }); } - // Tab after spaces is start of content, indent count is correct + // Tab after spaces, or mid-line: separation or content, indent count is correct } Ok(count) } @@ -3079,8 +3079,9 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { /// just past their closing delimiter, so the main loop can re-derive an "indent" /// from a mid-line cursor. A tab in the middle of a line is never indentation. /// - /// Only reachable with `indent >= 1`: `count_indent` already returns - /// `Err(TabIndentation)` for a tab at column 0, so `Ok(0)` implies no tab here. + /// `count_indent` returns `Err(TabIndentation)` for a tab at column 0 of a *line*, so an + /// `Ok(0)` with a tab at the cursor is a mid-line entry (#6937), which the `line_start` test + /// above rules out. #[inline] fn tab_indents_block_structure(&self, line_start: usize) -> bool { self.peek() == Some(b'\t') diff --git a/tests/yq_cli/i6937.rs b/tests/yq_cli/i6937.rs index c6b470804..63dcbeeda 100644 --- a/tests/yq_cli/i6937.rs +++ b/tests/yq_cli/i6937.rs @@ -30,6 +30,17 @@ const ACCEPTED: &[(&str, &str)] = &[ ("a: b\t\n", "{\"a\":\"b\"}\n"), ("[1]\t\n", "[1]\n"), ("a: {x: 1}\t\n", "{\"a\":{\"x\":1}}\n"), + ("a: \"b\"\t\r\n", "{\"a\":\"b\"}\n"), + ("\"a\"\t\r\n", "\"a\"\n"), + ("--- \"a\"\t\n", "\"a\"\n"), + ("---\n\"a\"\t\n---\n", "\"a\"\nnull\n"), + ("\"a\"\t\n...\n", "\"a\"\n"), + ("--- \"a\"\t\n\t\n", "\"a\"\n"), + ("- &a \"x\"\t\n- *a\t\n", "[\"x\",\"x\"]\n"), + ("? k: v\t\n", "{\"\":null}\n"), + ("- ? a\t\n", "[{\"a\":null}]\n"), + ("\"a\"\t:\n", "{\"a\":null}\n"), + ("\"a\"\t: 1\n", "{\"a\":1}\n"), ]; /// Refused by yq (a tab that starts a line, or content after the tab). @@ -42,6 +53,15 @@ const REFUSED: &[&str] = &[ "\"a\"\tb\n", "- \"a\"\t- b\n", "\"a\"\n \t\n", + "\"a\"\r\n\t\r\n", + "a: \"b\"\r\n\t\r\n", + "\"a\"\tb: 1\n", + "\"a\"\t- b\n", + "\"a\"\t? k\n", + "\"a\"\t\n\t\n", + "a: \"b\"\t\n \t\n", + "? \"a\"\t\n : 1\n", + "[ \"a\" ]\t\n\t\n", ]; #[test]