fix(yaml): read a tab after a closing quote or an alias as separation

changelog.d/6937.fixed.md

@@ -0,0 +1 @@
+- **yq: a tab right after a quoted scalar or an alias in block context is read as separation** (#6937). `a: "b"⇥`, `"a"⇥`, `- 'x'⇥`, `a: &x "b"⇥` and `a: *x⇥` were a parse error ("tab character used for indentation"), where yq reads them: the scanners for a quoted scalar and an alias stop just past their closing delimiter, so the main loop was entered mid-line and took the tab at the cursor for indentation, or for a line of its own. A tab that starts a line, and content after the tab (`a: "b"⇥c`), are still refused, as in yq. Found by a mutation fuzz of the yaml-test-suite inputs against yq v4.53.3.

src/yaml/parser.rs

@@ -3030,6 +3030,15 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> {
         }
     }
 
+    /// Whether the cursor is at the start of a line: the start of the input, or just past a break.
+    ///
+    /// The scanners for a quoted scalar, a flow collection and an alias stop just past their
+    /// closing delimiter, so the main loop can be entered mid-line (#6937).
+    #[inline]
+    fn at_line_start(&self) -> bool {
+        self.pos == 0 || Self::is_break(self.input[self.pos - 1])
+    }
+
     /// Count leading spaces (indentation) at start of a line.
     fn count_indent(&self) -> Result<usize, YamlError> {
         // Use SIMD-accelerated space counting
@@ -3041,13 +3050,16 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> {
             // Tab after spaces - check context
             // If we haven't seen any spaces and hit a tab at start of line,
             // that's tab indentation (error). But tab after spaces is content.
-            if count == 0 {
+            // #6937: only at the start of a line. The quoted-scalar and alias scanners stop just
+            // past their delimiter, and a tab there (`a: "b"⇥`) is separation before the line break
+            // or a comment, which yq reads.
+            if count == 0 && self.at_line_start() {
                 return Err(YamlError::TabIndentation {
                     line: self.current_line(),
                     offset: next_pos,
                 });
             }
-            // Tab after spaces is start of content, indent count is correct
+            // Tab after spaces, or mid-line: separation or content, indent count is correct
         }
         Ok(count)
     }
@@ -3067,8 +3079,9 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> {
     /// just past their closing delimiter, so the main loop can re-derive an "indent"
     /// from a mid-line cursor. A tab in the middle of a line is never indentation.
     ///
-    /// Only reachable with `indent >= 1`: `count_indent` already returns
-    /// `Err(TabIndentation)` for a tab at column 0, so `Ok(0)` implies no tab here.
+    /// `count_indent` returns `Err(TabIndentation)` for a tab at column 0 of a *line*, so an
+    /// `Ok(0)` with a tab at the cursor is a mid-line entry (#6937), which the `line_start` test
+    /// above rules out.
     #[inline]
     fn tab_indents_block_structure(&self, line_start: usize) -> bool {
         self.peek() == Some(b'\t')
@@ -3145,8 +3158,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> {
     /// past an empty value or key and would otherwise take the line as its blank prefix (#6295: `a:` /
     /// ` ⇥` is refused by yq, and was read as `{"a": null}`; `?` / ` ⇥` / `  a` as `{"a": null}` too).
     fn reject_tab_only_line_at_cursor(&self) -> Result<(), YamlError> {
-        let at_line_start = self.pos == 0 || Self::is_break(self.input[self.pos - 1]);
-        if at_line_start && self.tab_only_line_is_refused(self.pos) {
+        if self.at_line_start() && self.tab_only_line_is_refused(self.pos) {
             return Err(YamlError::TabIndentation {
                 line: self.current_line(),
                 offset: self.pos,
@@ -10211,6 +10223,7 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> {
 
         // Skip to content
         let line_start = self.pos;
+        let entered_at_line_start = self.at_line_start();
         self.advance_by(indent);
 
         // A tab *after* the leading spaces is indentation — and so illegal — only when
@@ -10225,7 +10238,11 @@ impl<'a, const HAS_CR: bool> Parser<'a, HAS_CR> {
         }
 
         // The cursor is past the line's spaces: only a line whose next byte is the tab can be the one.
-        if self.peek() == Some(b'\t') && self.tab_only_line_is_refused(line_start) {
+        // A tab after a closing quote or an alias is no line of its own (#6937).
+        if entered_at_line_start
+            && self.peek() == Some(b'\t')
+            && self.tab_only_line_is_refused(line_start)
+        {
             return Err(YamlError::TabIndentation {
                 line: self.current_line(),
                 offset: self.pos,

tests/yq_cli/i6937.rs

@@ -0,0 +1,84 @@
+//! #6937: a tab directly after the closing quote of a quoted scalar, or after an alias, in block context is
+//! separation before the line break or a comment, as it is after a plain scalar or a flow collection. The
+//! scanners for those stop just past their closing delimiter, so the main loop was entered mid-line and read
+//! the tab at the cursor as indentation. A tab that starts a line is still refused, and so is content after it.
+//!
+//! Every row is captured from yq v4.53.3 (`-o=json -I=0 .`).
+
+use anyhow::Result;
+
+use crate::run_yq_stdin;
+
+/// Accepted by yq: `(document, expected output)`.
+const ACCEPTED: &[(&str, &str)] = &[
+    ("\"a\"\t\n", "\"a\"\n"),
+    ("'a'\t\n", "\"a\"\n"),
+    ("a: \"b\"\t\n", "{\"a\":\"b\"}\n"),
+    ("a: \"b\"\t# c\n", "{\"a\":\"b\"}\n"),
+    ("a: \"b\"\t \n", "{\"a\":\"b\"}\n"),
+    ("a: \"b\" \t\n", "{\"a\":\"b\"}\n"),
+    ("- \"x\"\t\n- y\n", "[\"x\",\"y\"]\n"),
+    ("? \"a\"\t\n: 1\n", "{\"a\":1}\n"),
+    ("a: &x \"b\"\t\n", "{\"a\":\"b\"}\n"),
+    ("x: &x 1\na: *x\t\n", "{\"x\":1,\"a\":1}\n"),
+    ("a: !!str \"b\"\t\n", "{\"a\":\"b\"}\n"),
+    ("a: \"b\n  c\"\t\n", "{\"a\":\"b c\"}\n"),
+    ("\"a\"\t", "\"a\"\n"),
+    ("\"a\"\t\t\n", "\"a\"\n"),
+    ("a: \"b\"\t\nc: 1\n", "{\"a\":\"b\",\"c\":1}\n"),
+    ("a:\n  - \"b\"\t\n  - c\n", "{\"a\":[\"b\",\"c\"]}\n"),
+    ("a: b\t\n", "{\"a\":\"b\"}\n"),
+    ("[1]\t\n", "[1]\n"),
+    ("a: {x: 1}\t\n", "{\"a\":{\"x\":1}}\n"),
+    ("a: \"b\"\t\r\n", "{\"a\":\"b\"}\n"),
+    ("\"a\"\t\r\n", "\"a\"\n"),
+    ("--- \"a\"\t\n", "\"a\"\n"),
+    ("---\n\"a\"\t\n---\n", "\"a\"\nnull\n"),
+    ("\"a\"\t\n...\n", "\"a\"\n"),
+    ("--- \"a\"\t\n\t\n", "\"a\"\n"),
+    ("- &a \"x\"\t\n- *a\t\n", "[\"x\",\"x\"]\n"),
+    ("? k: v\t\n", "{\"\":null}\n"),
+    ("- ? a\t\n", "[{\"a\":null}]\n"),
+    ("\"a\"\t:\n", "{\"a\":null}\n"),
+    ("\"a\"\t: 1\n", "{\"a\":1}\n"),
+];
+
+/// Refused by yq (a tab that starts a line, or content after the tab).
+const REFUSED: &[&str] = &[
+    "\"a\"\n\t\n",
+    "a: \"b\"\n\t\n",
+    "\t\"a\"\n",
+    "a:\n\t\"b\"\n",
+    "a: \"b\"\tc\n",
+    "\"a\"\tb\n",
+    "- \"a\"\t- b\n",
+    "\"a\"\n \t\n",
+    "\"a\"\r\n\t\r\n",
+    "a: \"b\"\r\n\t\r\n",
+    "\"a\"\tb: 1\n",
+    "\"a\"\t- b\n",
+    "\"a\"\t? k\n",
+    "\"a\"\t\n\t\n",
+    "a: \"b\"\t\n  \t\n",
+    "? \"a\"\t\n  : 1\n",
+    "[ \"a\" ]\t\n\t\n",
+];
+
+#[test]
+fn a_tab_after_a_closing_quote_or_an_alias_is_separation_6937() -> Result<()> {
+    for (doc, want) in ACCEPTED {
+        let (out, code) = run_yq_stdin(".", doc, &["-o=json", "-I=0"])?;
+        assert_eq!(code, 0, "{doc:?}");
+        assert_eq!(out, *want, "{doc:?}");
+    }
+    Ok(())
+}
+
+#[test]
+fn a_tab_that_starts_a_line_or_precedes_content_is_still_refused_6937() -> Result<()> {
+    for doc in REFUSED {
+        let (_, code) = run_yq_stdin(".", doc, &["-o=json", "-I=0"])?;
+        assert_ne!(code, 0, "{doc:?}");
+    }
+    Ok(())
+}