diff --git a/changelog.d/6937.fixed.md b/changelog.d/6937.fixed.md new file mode 100644 index 000000000..f3ab24e98 --- /dev/null +++ b/changelog.d/6937.fixed.md @@ -0,0 +1 @@ +- **yq: a tab right after a quoted scalar or an alias in block context is read as separation** (#6937). `a: "b"⇥`, `"a"⇥`, `- 'x'⇥`, `a: &x "b"⇥` and `a: *x⇥` were a parse error ("tab character used for indentation"), where yq reads them: the scanners for a quoted scalar and an alias stop just past their closing delimiter, so the main loop was entered mid-line and took the tab at the cursor for indentation, or for a line of its own. A tab that starts a line, and content after the tab (`a: "b"⇥c`), are still refused, as in yq. Found by a mutation fuzz of the yaml-test-suite inputs against yq v4.53.3. diff --git a/src/yaml/parser.rs b/src/yaml/parser.rs index d7ff1ca60..9e58374e7 100644 --- a/src/yaml/parser.rs +++ b/src/yaml/parser.rs @@ -3030,6 +3030,15 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { } } + /// Whether the cursor is at the start of a line: the start of the input, or just past a break. + /// + /// The scanners for a quoted scalar, a flow collection and an alias stop just past their + /// closing delimiter, so the main loop can be entered mid-line (#6937). + #[inline] + fn at_line_start(&self) -> bool { + self.pos == 0 || Self::is_break(self.input[self.pos - 1]) + } + /// Count leading spaces (indentation) at start of a line. fn count_indent(&self) -> Result { // Use SIMD-accelerated space counting @@ -3041,13 +3050,16 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { // Tab after spaces - check context // If we haven't seen any spaces and hit a tab at start of line, // that's tab indentation (error). But tab after spaces is content. - if count == 0 { + // #6937: only at the start of a line. The quoted-scalar and alias scanners stop just + // past their delimiter, and a tab there (`a: "b"⇥`) is separation before the line break + // or a comment, which yq reads. + if count == 0 && self.at_line_start() { return Err(YamlError::TabIndentation { line: self.current_line(), offset: next_pos, }); } - // Tab after spaces is start of content, indent count is correct + // Tab after spaces, or mid-line: separation or content, indent count is correct } Ok(count) } @@ -3067,8 +3079,9 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { /// just past their closing delimiter, so the main loop can re-derive an "indent" /// from a mid-line cursor. A tab in the middle of a line is never indentation. /// - /// Only reachable with `indent >= 1`: `count_indent` already returns - /// `Err(TabIndentation)` for a tab at column 0, so `Ok(0)` implies no tab here. + /// `count_indent` returns `Err(TabIndentation)` for a tab at column 0 of a *line*, so an + /// `Ok(0)` with a tab at the cursor is a mid-line entry (#6937), which the `line_start` test + /// above rules out. #[inline] fn tab_indents_block_structure(&self, line_start: usize) -> bool { self.peek() == Some(b'\t') @@ -3145,8 +3158,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { /// past an empty value or key and would otherwise take the line as its blank prefix (#6295: `a:` / /// ` ⇥` is refused by yq, and was read as `{"a": null}`; `?` / ` ⇥` / ` a` as `{"a": null}` too). fn reject_tab_only_line_at_cursor(&self) -> Result<(), YamlError> { - let at_line_start = self.pos == 0 || Self::is_break(self.input[self.pos - 1]); - if at_line_start && self.tab_only_line_is_refused(self.pos) { + if self.at_line_start() && self.tab_only_line_is_refused(self.pos) { return Err(YamlError::TabIndentation { line: self.current_line(), offset: self.pos, @@ -10211,6 +10223,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { // Skip to content let line_start = self.pos; + let entered_at_line_start = self.at_line_start(); self.advance_by(indent); // A tab *after* the leading spaces is indentation — and so illegal — only when @@ -10225,7 +10238,11 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> { } // The cursor is past the line's spaces: only a line whose next byte is the tab can be the one. - if self.peek() == Some(b'\t') && self.tab_only_line_is_refused(line_start) { + // A tab after a closing quote or an alias is no line of its own (#6937). + if entered_at_line_start + && self.peek() == Some(b'\t') + && self.tab_only_line_is_refused(line_start) + { return Err(YamlError::TabIndentation { line: self.current_line(), offset: self.pos, diff --git a/tests/yq_cli/i6937.rs b/tests/yq_cli/i6937.rs new file mode 100644 index 000000000..63dcbeeda --- /dev/null +++ b/tests/yq_cli/i6937.rs @@ -0,0 +1,84 @@ +//! #6937: a tab directly after the closing quote of a quoted scalar, or after an alias, in block context is +//! separation before the line break or a comment, as it is after a plain scalar or a flow collection. The +//! scanners for those stop just past their closing delimiter, so the main loop was entered mid-line and read +//! the tab at the cursor as indentation. A tab that starts a line is still refused, and so is content after it. +//! +//! Every row is captured from yq v4.53.3 (`-o=json -I=0 .`). + +use anyhow::Result; + +use crate::run_yq_stdin; + +/// Accepted by yq: `(document, expected output)`. +const ACCEPTED: &[(&str, &str)] = &[ + ("\"a\"\t\n", "\"a\"\n"), + ("'a'\t\n", "\"a\"\n"), + ("a: \"b\"\t\n", "{\"a\":\"b\"}\n"), + ("a: \"b\"\t# c\n", "{\"a\":\"b\"}\n"), + ("a: \"b\"\t \n", "{\"a\":\"b\"}\n"), + ("a: \"b\" \t\n", "{\"a\":\"b\"}\n"), + ("- \"x\"\t\n- y\n", "[\"x\",\"y\"]\n"), + ("? \"a\"\t\n: 1\n", "{\"a\":1}\n"), + ("a: &x \"b\"\t\n", "{\"a\":\"b\"}\n"), + ("x: &x 1\na: *x\t\n", "{\"x\":1,\"a\":1}\n"), + ("a: !!str \"b\"\t\n", "{\"a\":\"b\"}\n"), + ("a: \"b\n c\"\t\n", "{\"a\":\"b c\"}\n"), + ("\"a\"\t", "\"a\"\n"), + ("\"a\"\t\t\n", "\"a\"\n"), + ("a: \"b\"\t\nc: 1\n", "{\"a\":\"b\",\"c\":1}\n"), + ("a:\n - \"b\"\t\n - c\n", "{\"a\":[\"b\",\"c\"]}\n"), + ("a: b\t\n", "{\"a\":\"b\"}\n"), + ("[1]\t\n", "[1]\n"), + ("a: {x: 1}\t\n", "{\"a\":{\"x\":1}}\n"), + ("a: \"b\"\t\r\n", "{\"a\":\"b\"}\n"), + ("\"a\"\t\r\n", "\"a\"\n"), + ("--- \"a\"\t\n", "\"a\"\n"), + ("---\n\"a\"\t\n---\n", "\"a\"\nnull\n"), + ("\"a\"\t\n...\n", "\"a\"\n"), + ("--- \"a\"\t\n\t\n", "\"a\"\n"), + ("- &a \"x\"\t\n- *a\t\n", "[\"x\",\"x\"]\n"), + ("? k: v\t\n", "{\"\":null}\n"), + ("- ? a\t\n", "[{\"a\":null}]\n"), + ("\"a\"\t:\n", "{\"a\":null}\n"), + ("\"a\"\t: 1\n", "{\"a\":1}\n"), +]; + +/// Refused by yq (a tab that starts a line, or content after the tab). +const REFUSED: &[&str] = &[ + "\"a\"\n\t\n", + "a: \"b\"\n\t\n", + "\t\"a\"\n", + "a:\n\t\"b\"\n", + "a: \"b\"\tc\n", + "\"a\"\tb\n", + "- \"a\"\t- b\n", + "\"a\"\n \t\n", + "\"a\"\r\n\t\r\n", + "a: \"b\"\r\n\t\r\n", + "\"a\"\tb: 1\n", + "\"a\"\t- b\n", + "\"a\"\t? k\n", + "\"a\"\t\n\t\n", + "a: \"b\"\t\n \t\n", + "? \"a\"\t\n : 1\n", + "[ \"a\" ]\t\n\t\n", +]; + +#[test] +fn a_tab_after_a_closing_quote_or_an_alias_is_separation_6937() -> Result<()> { + for (doc, want) in ACCEPTED { + let (out, code) = run_yq_stdin(".", doc, &["-o=json", "-I=0"])?; + assert_eq!(code, 0, "{doc:?}"); + assert_eq!(out, *want, "{doc:?}"); + } + Ok(()) +} + +#[test] +fn a_tab_that_starts_a_line_or_precedes_content_is_still_refused_6937() -> Result<()> { + for doc in REFUSED { + let (_, code) = run_yq_stdin(".", doc, &["-o=json", "-I=0"])?; + assert_ne!(code, 0, "{doc:?}"); + } + Ok(()) +}