From c8a481d573f7b9c6fbe6126136213dc22729dbd9 Mon Sep 17 00:00:00 2001 From: Kevin Burke Date: Fri, 22 May 2026 15:00:32 -0700 Subject: [PATCH 1/2] regex: accept literal [ in bracket expressions POSIX allows '[' to represent itself inside a bracket expression unless it starts a character class, collating symbol, or equivalence class construct. The parser was consuming the following character after a literal '[', which made '[[]' look unterminated, and the Rust regex backend also rejects unescaped literal '[' in classes. Reproduce the compatibility gap by comparing these commands: printf 'x\n' | ./target/debug/sed -E 's/[[]/X/' printf 'x\n' | gsed -E 's/[[]/X/' printf 'x\n' | ./target/debug/sed -E 's/[^[]/X/' printf 'x\n' | gsed -E 's/[^[]/X/' Before this change, uutils sed rejected the scripts as unterminated or invalid regexes while GNU sed parsed them cleanly. After this change the outputs match GNU sed: x, X, x, and X for the reported cases. Leave the following character available for normal class parsing, then escape literal '[' characters inside parsed classes before compiling with the regex backend. Add parser, compiler, and command-level regressions for the failing sed -E substitution cases. Observed this while trying to install Python 3.14.5 with Pyenv and uutils `sed` on the PATH, which failed. --- src/sed/compiler.rs | 128 ++++++++++++++++++++++++++++++++++++ src/sed/delimited_parser.rs | 37 ++++++++++- tests/by-util/test_sed.rs | 17 +++++ 3 files changed, 179 insertions(+), 3 deletions(-) diff --git a/src/sed/compiler.rs b/src/sed/compiler.rs index 0e7d5ca2..38047d32 100644 --- a/src/sed/compiler.rs +++ b/src/sed/compiler.rs @@ -655,6 +655,74 @@ fn bre_to_ere(pattern: &[u8]) -> Vec { result } +fn escape_literal_open_brackets_in_classes(pattern: &[u8]) -> Vec { + let mut result = Vec::with_capacity(pattern.len()); + let mut bytes = pattern.iter().copied().peekable(); + + while let Some(c) = bytes.next() { + match c { + b'\\' => { + result.push(b'\\'); + if let Some(escaped) = bytes.next() { + result.push(escaped); + } + continue; + } + b'[' => result.push(b'['), + _ => { + result.push(c); + continue; + } + } + + if bytes.peek() == Some(&b'^') { + result.push(b'^'); + bytes.next(); + } + + if bytes.peek() == Some(&b']') { + result.push(b']'); + bytes.next(); + } + + while let Some(class_byte) = bytes.next() { + match class_byte { + b']' => { + result.push(b']'); + break; + } + b'\\' => { + result.push(b'\\'); + if let Some(escaped) = bytes.next() { + result.push(escaped); + } + } + b'[' => { + if let Some(&marker @ (b':' | b'.' | b'=')) = bytes.peek() { + bytes.next(); + result.push(b'['); + result.push(marker); + + while let Some(posix_byte) = bytes.next() { + result.push(posix_byte); + if posix_byte == marker && bytes.peek() == Some(&b']') { + result.push(b']'); + bytes.next(); + break; + } + } + } else { + result.extend_from_slice(br"\["); + } + } + _ => result.push(class_byte), + } + } + } + + result +} + /// Compile the provided regular expression string into a corresponding engine. /// An empty pattern results in None, which means that the last RE employed /// at runtime will be used. @@ -677,6 +745,7 @@ fn compile_regex( } else { bre_to_ere(pattern) }; + let pattern = escape_literal_open_brackets_in_classes(&pattern); // Add any required modifiers. let mut modifiers = Vec::new(); @@ -1952,6 +2021,65 @@ mod tests { ); } + #[test] + fn test_compile_re_literal_open_bracket_in_classes() { + let (lines, chars) = dummy_providers(); + let mut context = ctx(); + context.regex_extended = true; + + for (pattern, matching, non_matching) in [ + ("[[]", "[", "x"), + ("[^[]", "x", "["), + ("[a[b]", "[", "x"), + ("[^a[b]", "x", "["), + ] { + let regex = compile_regex(&lines, &chars, pattern, &context, false, false) + .unwrap() + .expect("regex should be present"); + assert!( + regex + .is_match(&mut IOChunk::new_from_str(matching)) + .unwrap(), + "{pattern:?} should match {matching:?}" + ); + assert!( + !regex + .is_match(&mut IOChunk::new_from_str(non_matching)) + .unwrap(), + "{pattern:?} should not match {non_matching:?}" + ); + } + } + + #[test] + fn test_compile_re_escaped_open_bracket_before_class() { + let (lines, chars) = dummy_providers(); + let mut context = ctx(); + context.regex_extended = true; + + let regex = compile_regex(&lines, &chars, r"\[[a]", &context, false, false) + .unwrap() + .expect("regex should be present"); + assert!(regex.is_match(&mut IOChunk::new_from_str("[a")).unwrap()); + assert!(!regex.is_match(&mut IOChunk::new_from_str("[b")).unwrap()); + } + + #[test] + fn test_escape_literal_open_brackets_preserves_class_syntax() { + for (pattern, expected) in [ + (r"[a\]b]", r"[a\]b]"), + (r"[[:alpha:][x]", r"[[:alpha:]\[x]"), + (r"[[=a=][x]", r"[[=a=]\[x]"), + (r"[[.ch.][x]", r"[[.ch.]\[x]"), + ] { + assert_eq!( + escape_literal_open_brackets_in_classes(pattern.as_bytes()), + expected.as_bytes(), + "{pattern:?}" + ); + } + } + // compile_address #[test] fn test_compile_addr_line_number() { diff --git a/src/sed/delimited_parser.rs b/src/sed/delimited_parser.rs index b6a9b2d0..383effdd 100644 --- a/src/sed/delimited_parser.rs +++ b/src/sed/delimited_parser.rs @@ -304,10 +304,9 @@ fn parse_character_class( continue; } - // Not a POSIX construct — treat as literal + // Not a POSIX construct: '[' is literal, and the next character + // may still terminate or otherwise participate in the class. result.push(b'['); - result.push(line.current_byte()); - line.advance(); continue; } @@ -965,6 +964,38 @@ mod tests { assert_eq!(result, b"[^]abc]"); } + #[test] + fn test_literal_open_bracket() { + let mut line = char_provider_from("[[]"); + let lines = test_lines(); + let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap(); + assert_eq!(result, b"[[]"); + } + + #[test] + fn test_negated_literal_open_bracket() { + let mut line = char_provider_from("[^[]"); + let lines = test_lines(); + let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap(); + assert_eq!(result, b"[^[]"); + } + + #[test] + fn test_literal_open_bracket_in_class() { + let mut line = char_provider_from("[a[b]"); + let lines = test_lines(); + let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap(); + assert_eq!(result, b"[a[b]"); + } + + #[test] + fn test_negated_literal_open_bracket_in_class() { + let mut line = char_provider_from("[^a[b]"); + let lines = test_lines(); + let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap(); + assert_eq!(result, b"[^a[b]"); + } + #[test] fn test_escaped_character_begin() { let mut line = char_provider_from("[\\nabc]"); diff --git a/tests/by-util/test_sed.rs b/tests/by-util/test_sed.rs index 59b28d6f..932feec3 100644 --- a/tests/by-util/test_sed.rs +++ b/tests/by-util/test_sed.rs @@ -568,6 +568,23 @@ fn subst_multiline_flag_matches_embedded_line_end() { .stdout_is("foX\nbaX\n"); } +#[test] +fn test_subst_literal_open_bracket_in_character_classes() { + for (script, input, expected) in [ + (r"s/[[]/X/", "x\n", "x\n"), + (r"s/[^[]/X/", "x\n", "X\n"), + (r"s/[a[b]/X/", "x\n", "x\n"), + (r"s/[^a[b]/X/", "x\n", "X\n"), + (r"s/\[[a]/X/", "[a\n", "X\n"), + ] { + new_ucmd!() + .args(&["-E", script]) + .pipe_in(input) + .succeeds() + .stdout_is(expected); + } +} + // Check appropriate selection and behavior of fast_Regex matcher // Literal matcher check_output!(subst_literal_start, ["-e", r"s/^l1/L1/", LINES1]); From 4bb5fa01c6900f6586bbb1879e51769bd5fa9168 Mon Sep 17 00:00:00 2001 From: Kevin Burke Date: Fri, 5 Jun 2026 12:02:31 -0700 Subject: [PATCH 2/2] sed/compiler: add function rustdoc --- src/sed/compiler.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sed/compiler.rs b/src/sed/compiler.rs index 38047d32..3b32618a 100644 --- a/src/sed/compiler.rs +++ b/src/sed/compiler.rs @@ -655,6 +655,14 @@ fn bre_to_ere(pattern: &[u8]) -> Vec { result } +/// Escape literal `[` characters that appear inside a bracket expression. +/// +/// Within a bracket expression a `[` only begins a sub-construct when followed +/// by `:`, `.`, or `=` (e.g. `[:alpha:]`); elsewhere it is an ordinary +/// character. The `regex` crate rejects such a bare `[`, so we escape those +/// occurrences (`[` becomes `\[`) before handing the pattern to the engine. See +/// POSIX 9.3.5 RE Bracket Expression: +/// fn escape_literal_open_brackets_in_classes(pattern: &[u8]) -> Vec { let mut result = Vec::with_capacity(pattern.len()); let mut bytes = pattern.iter().copied().peekable();