diff --git a/CHANGELOG.md b/CHANGELOG.md index b56e49b5..c1d53389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -307,6 +307,19 @@ because it turns other people's test suites red. ### Added +- **A preset for unusual file names: `filename-handling`.** It answers "will + my system store, show and give back a file name it did not expect?" with + fifty names in seven groups: scripts from Polish to Korean, names that look + like other names, leading spaces and dots, shell and SQL metacharacters, + names that mean something to a web server or a desktop, names read as + values, and names at the length limits. Every one is written byte for byte + on Windows, Linux and macOS - measured on NTFS, ext4 and APFS. On another + file system a name may be refused, and the run then ends with code 8 and + names it. The files are `txt` unless `--format` says otherwise, and the + names about length count the format's extension in. Four names are + expected to be accepted, the rest are left to your system's policy with a + reason. + - **Every format has its full name.** `tfg formats` has a `NAME` column (`jxl` is JPEG XL, `png` Portable Network Graphics), `tfg formats jxl` gives it on a `name` line, and `tfg formats --json` carries it under the new @@ -507,6 +520,32 @@ because it turns other people's test suites red. ### Fixed +- **A report shows a character nobody can see in a file name as an escape.** + `verify`, `cleanup`, the notes of a run, every error message and the + refusals in the window printed such a character as it was, in a file name + and in the name of a folder. A right to left override then made the terminal draw + another name than the one on the disk, and a zero width space made two + names look the same. Such a character is printed as an escape now, such as + `\u202e` for a right to left override. A name without one is printed + as before. The manifest and every `--json` report still carry the + exact name. + +- **A recipe the tool writes shows a character nobody can see as an escape.** + `tfg preset eject` wrote a right to left override, a zero width space or a + line separator into the recipe as it was, so the file read as something + other than what it held, and a YAML 1.1 reader such as PyYAML refused it at + the line separator. Such a value is written in double quotes now, with the + character as an escape that every YAML reader turns back into the same + character. Every other value is written as before. A run started in the + window records a different recipe hash only when a value holds such a + character. + +- **A space from outside ASCII at the start or the end of a recipe value is + kept.** A file name beginning with an ideographic space or a no break space + lost it, and the file was written under a different name than the recipe + asked for, with nothing said. A plain space or a tab at the ends of an + unquoted value is still not part of it, as in any YAML file. + - **A file name from 238 to 255 bytes long is written.** Every system stores such a name, and none of them got one: each file is written under a longer temporary name first, and that one was over the limit. The same held for a diff --git a/README.md b/README.md index 7a9dca3a..81ed10fb 100644 --- a/README.md +++ b/README.md @@ -639,7 +639,8 @@ when a number is a placeholder of ours rather than a limit of yours. Presets are ordinary recipes underneath - `tfg preset eject size-boundaries` prints the recipe and you edit it from there. -One preset ships today, `size-boundaries`. More are designed. +`tfg preset list` names every preset your build ships, and the Presets +screen of the window offers the same ones. ## 🖥️ The desktop window diff --git a/internal/audit/audit.go b/internal/audit/audit.go index 7591f1d1..890a6ea8 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -110,6 +110,9 @@ type Difference struct { } func (d Difference) String() string { + // Shown on a copy, so the paths a program compares stay as they are. See + // core.Shown for the name this printed wrongly (O241). + d.Path, d.Want, d.Got = core.Shown(d.Path), core.Shown(d.Want), core.Shown(d.Got) switch d.Kind { case Missing: return fmt.Sprintf("missing %s", d.Path) @@ -191,7 +194,7 @@ func (e *EscapeError) Error() string { "the manifest lists %q, which lands outside %s once the links on the way are followed. "+ "This tool never reads or removes anything outside the directory it was pointed at, so it will not act on this manifest. "+ "Check that the directory is the one the run wrote to, and that nothing inside it points elsewhere.", - e.Path, e.Dir) + e.Path, core.Shown(e.Dir)) } // resolved turns a manifest entry into the path on disk, refusing one that diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index b07aea52..536e4ecf 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -117,13 +117,13 @@ func previewCleanup(cands []audit.Candidate, path, dir string, force, asJSON boo return writeJSON(out, errOut, report, ExitOK) } - fmt.Fprintf(out, "%s would be removed from %s:\n", core.Count(countRemovable(cands, force), "file", "files"), dir) + fmt.Fprintf(out, "%s would be removed from %s:\n", core.Count(countRemovable(cands, force), "file", "files"), core.Shown(dir)) for _, c := range cands { if c.Removable(force) { - fmt.Fprintf(out, " remove %s\n", c.Path) + fmt.Fprintf(out, " remove %s\n", core.Shown(c.Path)) continue } - fmt.Fprintf(out, " keep %s - %s\n", c.Path, skipNote(c, force)) + fmt.Fprintf(out, " keep %s - %s\n", core.Shown(c.Path), skipNote(c, force)) } fmt.Fprintf(errOut, "Nothing was removed. Run the same command with --yes to remove them.\n") return ExitOK @@ -153,7 +153,7 @@ func applyCleanup(ctx context.Context, cands []audit.Candidate, path, dir string } report.Files = append(report.Files, cleanupEntry{Path: o.Path, Action: "kept", Reason: o.Reason}) if !asJSON { - fmt.Fprintf(errOut, "kept %s - %s\n", o.Path, o.Reason) + fmt.Fprintf(errOut, "kept %s - %s\n", core.Shown(o.Path), core.Shown(o.Reason)) } } // Kept counts every entry that is not removed, which is what the entries @@ -179,7 +179,7 @@ func applyCleanup(ctx context.Context, cands []audit.Candidate, path, dir string if blocked > 0 { fmt.Fprintf(errOut, "tfg: the manifest was kept. It is the only record of %s still on disk.\n", core.Count(blocked, "file", "files")) } else if err := os.Remove(path); err != nil { - fmt.Fprintf(errOut, "tfg: cannot remove the manifest %s: %s\n", path, describeError(err)) + fmt.Fprintf(errOut, "tfg: cannot remove the manifest %s: %s\n", core.Shown(path), describeError(err)) return ExitIO } } @@ -194,7 +194,7 @@ func applyCleanup(ctx context.Context, cands []audit.Candidate, path, dir string return writeJSON(out, errOut, report, ExitOK) } - fmt.Fprintf(out, "%s removed from %s\n", core.Count(removed, "file", "files"), dir) + fmt.Fprintf(out, "%s removed from %s\n", core.Count(removed, "file", "files"), core.Shown(dir)) // A file left behind is not a silent outcome. It was reported above, and // the exit code has to carry it too or a script never learns. diff --git a/internal/cli/errors.go b/internal/cli/errors.go index 2058e0ea..9ed1b352 100644 --- a/internal/cli/errors.go +++ b/internal/cli/errors.go @@ -12,6 +12,7 @@ import ( "syscall" "github.com/donislawdev/TestingFilesGenerator/internal/audit" + "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/damage" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/format" @@ -33,7 +34,19 @@ import ( // So the system's sentence is swapped for ours and every layer of our own // context above it is kept. The number it carried stays, because a number means // the same thing in every language and it is what somebody puts into a search. +// +// And a character nobody can see is shown rather than left to act, in every +// message at once (O241). A system error wrapped under our own sentence +// repeats the path it failed on in its own words, raw, after our sentence had +// shown it escaped - found in a review on 2026-09-25. One funnel covers every +// command, including the ones that print a message nobody wrote with a name +// in mind. func describeError(err error) string { + return core.ShownText(inOurWords(err)) +} + +// inOurWords is describeError before anything is escaped. +func inOurWords(err error) string { if err == nil { return "" } diff --git a/internal/cli/generate.go b/internal/cli/generate.go index fd3d674d..50e25bfd 100644 --- a/internal/cli/generate.go +++ b/internal/cli/generate.go @@ -564,7 +564,7 @@ func echoBoundaries(targets []engine.Target, planned []engine.PlannedFile, errOu fmt.Fprintf(errOut, "boundary %q around %s:\n", t.ID, core.ExactBytes(t.BoundaryLimit)) for _, f := range planned { if f.Target == t { - fmt.Fprintf(errOut, " %-26s %s\n", f.Name, core.ExactBytes(f.Plan.Bytes)) + fmt.Fprintf(errOut, " %-26s %s\n", core.Shown(f.Name), core.ExactBytes(f.Plan.Bytes)) } } @@ -656,7 +656,7 @@ func saveManifest(res *engine.Result, opt engine.Options, errOut io.Writer) int // way is a chance for the saver and the claim to mean different files. path := engine.ManifestPath(opt) if err := res.Manifest.Save(path); err != nil { - fmt.Fprintf(errOut, "tfg: cannot write the manifest to %s: %s\n", path, describeError(err)) + fmt.Fprintf(errOut, "tfg: cannot write the manifest to %s: %s\n", core.Shown(path), describeError(err)) // What that leaves behind, because the line above is about the manifest // and the person's problem is the files. Rule 6: a run that wrote files // nothing can remove says so rather than leaving it to be discovered by @@ -667,10 +667,10 @@ func saveManifest(res *engine.Result, opt engine.Options, errOut io.Writer) int if n := len(res.Manifest.Files); n > 0 { fmt.Fprintf(errOut, "tfg: %s written and nothing to record what this run left. Cleanup works from a manifest, so clearing %s is a job by hand.\n", - core.Count(n, "file", "files"), opt.OutDir) + core.Count(n, "file", "files"), core.Shown(opt.OutDir)) } return ExitIO } - fmt.Fprintf(errOut, "manifest: %s\n", path) + fmt.Fprintf(errOut, "manifest: %s\n", core.Shown(path)) return ExitOK } diff --git a/internal/cli/verify.go b/internal/cli/verify.go index aff76db7..5b207ac1 100644 --- a/internal/cli/verify.go +++ b/internal/cli/verify.go @@ -70,7 +70,7 @@ Flags: dir = filepath.Dir(path) } if info, statErr := os.Stat(dir); statErr != nil || !info.IsDir() { - fmt.Fprintf(errOut, "tfg: cannot read the directory %s. Check the path and that you have permission to read it.\n", dir) + fmt.Fprintf(errOut, "tfg: cannot read the directory %s. Check the path and that you have permission to read it.\n", core.Shown(dir)) return ExitIO } @@ -150,7 +150,7 @@ func reportVerify(diffs []audit.Difference, claimed int, path, dir string, asJSO } if wrong > 0 { - fmt.Fprintf(errOut, "tfg: %s does not match %s - %s:\n", dir, path, core.Count(wrong, "difference", "differences")) + fmt.Fprintf(errOut, "tfg: %s does not match %s - %s:\n", core.Shown(dir), core.Shown(path), core.Count(wrong, "difference", "differences")) echoMismatches(diffs, errOut) echoOtherRuns(diffs, errOut) return ExitVerify @@ -160,11 +160,11 @@ func reportVerify(diffs []audit.Difference, claimed int, path, dir string, asJSO // "everything is fine" about zero files invites somebody to trust a run // that never happened. if claimed == 0 { - fmt.Fprintf(errOut, "%s claims no files, so there was nothing to check.\n", path) + fmt.Fprintf(errOut, "%s claims no files, so there was nothing to check.\n", core.Shown(path)) echoOtherRuns(diffs, errOut) return ExitOK } - fmt.Fprintf(out, "%s matches %s: %s checked\n", dir, path, core.Count(claimed, "file", "files")) + fmt.Fprintf(out, "%s matches %s: %s checked\n", core.Shown(dir), core.Shown(path), core.Count(claimed, "file", "files")) echoOtherRuns(diffs, errOut) return ExitOK } @@ -215,11 +215,11 @@ func echoOtherRuns(diffs []audit.Difference, errOut io.Writer) { for _, name := range names { files := byRecord[name] if len(files) == 0 { - fmt.Fprintf(errOut, "note: %s is another run's record, and nothing else here belongs to it.\n", name) + fmt.Fprintf(errOut, "note: %s is another run's record, and nothing else here belongs to it.\n", core.Shown(name)) continue } fmt.Fprintf(errOut, "note: %s is another run's record. %s here %s to it: %s.\n", - name, core.Count(len(files), "file", "files"), belongs(len(files)), someOf(files)) + core.Shown(name), core.Count(len(files), "file", "files"), belongs(len(files)), someOf(files)) } } @@ -249,6 +249,7 @@ func groupedByRecord(diffs []audit.Difference) map[string][]string { // someOf names the first few and counts the rest. func someOf(names []string) string { + names = core.ShownEach(names) if len(names) <= otherRunExamples { return strings.Join(names, ", ") } diff --git a/internal/core/unseen.go b/internal/core/unseen.go new file mode 100644 index 00000000..1c0aa497 --- /dev/null +++ b/internal/core/unseen.go @@ -0,0 +1,107 @@ +package core + +import ( + "strconv" + "strings" + "unicode/utf8" +) + +// Shown is s the way a person should read it: every character HoldsUnseen +// finds, and every byte that is not UTF-8, written as the escape %q would use +// for it, and nothing else changed. No quotes are added. +// +// For every line this tool prints about a name or a path that came from a +// recipe, a preset, a manifest or a directory listing (O241). Measured on +// 2026-09-24: verify reported a missing "photo", right to left override, +// "gpj.txt" as the terminal drew it, which is "phototxt.jpg", and an extra +// "in", zero width space, "voice.txt" as "invoice.txt" - a report naming files +// other than the ones on the disk, two of which could not be told apart. +// +// Without quotes, because a name holding nothing of the kind comes out byte +// for byte as it always did, and every report line of every run that never +// met such a name stays what scripts and people already read. The escape is +// not ambiguous inside a file name: a backslash is refused in one on every +// system (engine/filename.go). In a Windows path it reads as a separator +// followed by a letter and a number, which a person does not mistake for one. +// +// Never for what a program reads. The manifest and every --json report carry +// the name exactly, because a program compares it byte for byte. +func Shown(s string) string { + if !HoldsUnseen(s) && utf8.ValidString(s) { + return s + } + return shown(s, false) +} + +// ShownText is Shown for a whole message rather than one name: the line +// breaks and tabs it is laid out with stay as they are, and everything else +// nobody can see is escaped. +// +// For the places a message is turned into words for a person - the command +// line's describeError and the window's refusals - because an error wrapped +// from the operating system repeats the path it failed on in its own words, +// after this tool's sentence has already shown it. Measured on 2026-09-25, +// from a review: "cannot create the output directory" showed the folder +// escaped and the "mkdir" part after it showed it raw. +func ShownText(s string) string { + return shown(s, true) +} + +func shown(s string, layout bool) string { + var b strings.Builder + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + switch { + case r == utf8.RuneError && size == 1: + q := strconv.Quote(s[i : i+1]) + b.WriteString(q[1 : len(q)-1]) + case layout && (r == '\n' || r == '\t'): + b.WriteRune(r) + case !strconv.IsPrint(r): + q := strconv.QuoteRune(r) + b.WriteString(q[1 : len(q)-1]) + default: + b.WriteString(s[i : i+size]) + } + i += size + } + return b.String() +} + +// ShownEach is Shown for every name of a list, for the lines that name a few +// files one after another. +func ShownEach(names []string) []string { + out := make([]string, len(names)) + for i, name := range names { + out[i] = Shown(name) + } + return out +} + +// HoldsUnseen reports whether s holds a character a person reading it cannot +// see: a character that changes the direction of the text around it, one of +// no width, a byte order mark, a separator that breaks a line without being a +// line break, a space that is not the space bar's, a tag character, and every +// other one Go does not count as printable. +// +// It exists because file names are exactly where such characters are put on +// purpose. A name with a right to left override shows its extension in the +// wrong place, one with a zero width space prints as a name it is not, and +// both are test cases this tool writes (docs/NAMES-PRESET-2026-09-24.md). The +// file keeps its name. What a person reads about it has to show the character +// rather than let it act (O241), and a recipe has to carry it in a form that +// can be read and edited (O244). +// +// The class is strconv.IsPrint turned around, and that is a choice: it is the +// class %q escapes, and the refusals of this tool have quoted names with %q +// all along. One rule means one name looks the same in a refusal, in a report +// and in a recipe. A combining mark is printable and stays as it is. The +// space is printable, every other space is not. +func HoldsUnseen(s string) bool { + for _, r := range s { + if !strconv.IsPrint(r) { + return true + } + } + return false +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 976d74f4..e5f5c1e6 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -491,7 +491,7 @@ func Run(ctx context.Context, files []PlannedFile, opt Options) (*Result, error) } if err := os.MkdirAll(opt.OutDir, 0o755); err != nil { - return res, fmt.Errorf("cannot create the output directory %s: %w", opt.OutDir, err) + return res, fmt.Errorf("cannot create the output directory %s: %w", core.Shown(opt.OutDir), err) } // The directory is taken before the manifest name is, and the two are not @@ -511,7 +511,7 @@ func Run(ctx context.Context, files []PlannedFile, opt Options) (*Result, error) if errors.Is(err, fs.ErrExist) { return res, &RunInProgressError{Path: lockPath, Dir: opt.OutDir} } - return res, fmt.Errorf("cannot start a run in %s: %w", opt.OutDir, err) + return res, fmt.Errorf("cannot start a run in %s: %w", core.Shown(opt.OutDir), err) } // Given back however this run ends, including one stopped part way: the // signal cancels the context, Run returns, and this runs. What it cannot @@ -538,7 +538,7 @@ func Run(ctx context.Context, files []PlannedFile, opt Options) (*Result, error) if errors.Is(err, fs.ErrExist) { return res, &CollisionError{Path: manifestPath, Manifest: true} } - return res, fmt.Errorf("cannot start a run in %s: %w", opt.OutDir, err) + return res, fmt.Errorf("cannot start a run in %s: %w", core.Shown(opt.OutDir), err) } // Past this point the run owns the name and may write. Started says so, and diff --git a/internal/engine/errors.go b/internal/engine/errors.go index a53da958..49ef0006 100644 --- a/internal/engine/errors.go +++ b/internal/engine/errors.go @@ -205,7 +205,7 @@ type SpaceError struct { func (e *SpaceError) Error() string { return fmt.Sprintf( "this run needs %d B and %s has %d B free - nothing was written. Ask for fewer files or a smaller size, or write to another disk by changing the output directory", - e.Needed, e.Path, e.Available) + e.Needed, core.Shown(e.Path), e.Available) } // RunInProgressError is refusing to start because another run holds this @@ -228,7 +228,7 @@ type RunInProgressError struct { func (e *RunInProgressError) Error() string { return fmt.Sprintf( "another run is already writing into %s, so this one will not start. Two runs writing into one directory can write over each other's files without either of them saying so. Wait for it to finish, or generate into a different directory. If nothing is running, that run was killed before it could tidy up - remove %s and try again", - e.Dir, e.Path) + core.Shown(e.Dir), core.Shown(e.Path)) } // CollisionError is refusing to write over something that is already there. @@ -249,9 +249,9 @@ func (e *CollisionError) Error() string { if e.Manifest { return fmt.Sprintf( "%s already exists and this run will not write over it. It is the only record of what an earlier run wrote, so replacing it would leave those files with nothing to remove them by. Generate into an empty directory, or move the old manifest aside", - e.Path) + core.Shown(e.Path)) } return fmt.Sprintf( "%s already exists and this run will not write over it. Generate into an empty directory, or remove the file first", - e.Path) + core.Shown(e.Path)) } diff --git a/internal/engine/names.go b/internal/engine/names.go index 80a6c733..16324229 100644 --- a/internal/engine/names.go +++ b/internal/engine/names.go @@ -49,7 +49,7 @@ func claimFileName(names map[string]nameOwner, position int, id, name string) er return &RecipeError{ Setting: core.TargetAddress(position, SettingName), Detail: fmt.Sprintf("target %q produces a file named %s, and that is the name this run gives its manifest", - id, name), + id, core.Shown(name)), Because: "both are written into the output directory, so the file would take the name the manifest needs and the run would end with files and nothing to remove them by", Remedy: "Give the target a name template containing " + indexToken + ", or name the manifest something else", } @@ -134,13 +134,13 @@ func collisionKey(name string) string { func collisionDetail(owner nameOwner, id, name string) string { switch { case owner.name == name: - return fmt.Sprintf("targets %q and %q both produce a file named %s", owner.id, id, name) + return fmt.Sprintf("targets %q and %q both produce a file named %s", owner.id, id, core.Shown(name)) // Spelling before case, because normalising does not touch case and so a // pair that survives this one really is a difference of case. case norm.NFC.String(owner.name) == norm.NFC.String(name): return fmt.Sprintf( "targets %q and %q produce the names %s and %s. Those print the same because they are one name spelled two ways, an accented letter against the plain letter with its accent as a separate character. macOS stores both under one name, so one file would be written over the other and the manifest would describe both", - owner.id, id, owner.name, name) + owner.id, id, core.Shown(owner.name), core.Shown(name)) // Lowercasing rather than strings.EqualFold, and a guard caught the // difference on 2026-08-26. EqualFold folds simply, which puts the LONG s // in the same orbit as s - so "maſs.txt" against "mass.txt" was answered @@ -152,7 +152,7 @@ func collisionDetail(owner nameOwner, id, name string) string { case strings.ToLower(norm.NFC.String(owner.name)) == strings.ToLower(norm.NFC.String(name)): return fmt.Sprintf( "targets %q and %q produce the names %s and %s, which differ only in case. Most filesystems treat those as one file, so one would be written over the other and the manifest would describe both", - owner.id, id, owner.name, name) + owner.id, id, core.Shown(owner.name), core.Shown(name)) default: // The fourth kind, unreachable until collisionKey started folding on // 2026-08-26. It is not a difference of case and not a difference of @@ -161,6 +161,6 @@ func collisionDetail(owner nameOwner, id, name string) string { // accent, which is worse than saying nothing. return fmt.Sprintf( "targets %q and %q produce the names %s and %s. Those are different letters that mean the same one - the sharp s against ss, the long s against s, a ligature against the letters in it. macOS stores both under one name, so one file would be written over the other and the manifest would describe both", - owner.id, id, owner.name, name) + owner.id, id, core.Shown(owner.name), core.Shown(name)) } } diff --git a/internal/engine/preflight.go b/internal/engine/preflight.go index f25608f0..12958e0f 100644 --- a/internal/engine/preflight.go +++ b/internal/engine/preflight.go @@ -50,7 +50,7 @@ func preflight(ctx context.Context, files []PlannedFile, opt Options) error { // "missing", "no permission" and "already there". if info, err := os.Stat(opt.OutDir); err == nil && !info.IsDir() { return &RecipeError{Setting: SettingOutDir, - Detail: fmt.Sprintf("the output directory %s is a file, not a directory", opt.OutDir), + Detail: fmt.Sprintf("the output directory %s is a file, not a directory", core.Shown(opt.OutDir)), Remedy: "Point the output directory at a directory, or at one that does not exist yet and it will be created"} } diff --git a/internal/guard/filenamehandling_test.go b/internal/guard/filenamehandling_test.go new file mode 100644 index 00000000..b84ecdc4 --- /dev/null +++ b/internal/guard/filenamehandling_test.go @@ -0,0 +1,277 @@ +package guard + +import ( + "bytes" + "context" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "unicode/utf8" + + "golang.org/x/text/unicode/norm" + + "github.com/donislawdev/TestingFilesGenerator/internal/cli" + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/gui/text" + "github.com/donislawdev/TestingFilesGenerator/internal/manifest" + "github.com/donislawdev/TestingFilesGenerator/internal/preset" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// The preset of unusual file names asks for the fifty names of +// docs/NAMES-PRESET-2026-09-24.md, and each of them in every format. +// +// A name here is the whole point of its file, and a name that arrives a +// character short is a file that tests something else while its manifest +// entry still says what it was meant to test. That happened before this +// preset existed: an ideographic space at the front of a name was trimmed on +// its way through the recipe, in every format (O243). So each name is asked +// after the recipe has been written and read back, which is the name a run +// actually gets - and each is asked by what makes it the name it is, worked out +// here rather than copied from the preset. +func TestTheFileNamePresetAsksForEveryNameInEveryFormat(t *testing.T) { + checked := 0 + for _, id := range format.IDs() { + desc, _ := format.Get(id) + names := namesOfTheSet(t, preset.Args{"format": id}) + if len(names) != 50 { + t.Errorf("%s: the set holds %d names and the list has 50", id, len(names)) + } + folded := map[string]string{} + for target, name := range names { + if !utf8.ValidString(name) || name == "" || len(name) > core.MaxNameBytes { + t.Errorf("%s: %s is %d bytes, or empty, or not UTF-8: %+q", id, target, len(name), name) + } + if other, taken := folded[core.FoldName(name)]; taken { + t.Errorf("%s: %s and %s are one file on a system that folds names", id, target, other) + } + folded[core.FoldName(name)] = target + if fault := nameFault(target, name, desc.Extension); fault != "" { + t.Errorf("%s: %s is %+q, and %s", id, target, name, fault) + } + checked++ + } + } + if checked < 50*20 { + t.Fatalf("only %d names were checked", checked) + } + t.Logf("%d names in %d formats", checked, len(format.IDs())) +} + +// namesOfTheSet expands the preset, reads the recipe back and gives each +// target's name by its id. +func namesOfTheSet(t *testing.T, args preset.Args) map[string]string { + t.Helper() + expanded, err := preset.Expand("filename-handling", args) + if err != nil { + t.Fatalf("the preset refused %v: %v", args, err) + } + rec, err := recipe.Parse(expanded.Source, "filename-handling") + if err != nil { + t.Fatalf("the preset's recipe at %v was refused: %v", args, err) + } + out := map[string]string{} + for _, target := range rec.Targets { + out[target.ID] = target.Name + } + return out +} + +// nameFault says what is wrong with a name of the set, or "" when nothing is. +func nameFault(target, name, ext string) string { + exact := map[string]string{ + "htaccess": ".htaccess", "web_config": "web.config", "dotenv": ".env", + "ds_store": ".DS_Store", "desktop_ini": "desktop.ini", "no_extension": "README", + } + marked := map[string]rune{ + "bidi_override": 0x202E, "zero_width": 0x200B, "no_break_space": 0xA0, + "homoglyph": 0x430, "line_separator": 0x2028, "emoji_zwj": 0x200D, + } + stem := strings.TrimSuffix(name, ext) + switch { + case exact[target] != "": + if name != exact[target] { + return "it means something only as " + exact[target] + } + return "" + case target == "upper_extension": + if name != "REPORT"+strings.ToUpper(ext) { + return "its extension is not the format's in capitals" + } + return "" + case target == "fullwidth_extension": + if name == "report"+ext || norm.NFKC.String(name) != "report"+ext { + return "it is not the format's extension in full width letters" + } + return "" + case !strings.HasSuffix(name, ext): + return "it does not end with the format's extension " + ext + } + switch target { + case "only_extension": + return unless(stem == "", "it is more than the extension") + case "ustar_101": + return unless(len(name) == 101 && strings.Trim(stem, "u") == "", "it is not 101 bytes of u with the extension") + case "max_ascii": + return unless(len(name) == 255 && strings.Trim(stem, "a") == "", "it is not 255 bytes of a with the extension") + case "cjk_bytes": + return unless(strings.Trim(stem, "日") == "" && len(name) <= 255 && len(name)+3 > 255, "it is not as many ideographs as fit in 255 bytes") + case "emoji_bytes": + return unless(strings.Trim(stem, "🎉") == "" && len(name) <= 255 && len(name)+4 > 255, "it is not as many emoji as fit in 255 bytes") + case "nfd", "hangul_nfd": + return unless(norm.NFD.IsNormalString(name) && !norm.NFC.IsNormalString(name), "it is not in the decomposed form") + case "leading_bom": + return unless(strings.HasPrefix(name, string(rune(0xFEFF))), "it does not begin with a byte order mark") + case "leading_space": + return unless(strings.HasPrefix(name, " ") && !strings.HasPrefix(name, " "), "it does not begin with one plain space") + case "double_space": + return unless(strings.Contains(stem, " "), "it does not hold two spaces in a row") + case "leading_dot": + return unless(strings.HasPrefix(name, ".") && !strings.HasPrefix(name, ".."), "it does not begin with one dot") + case "leading_double_dot": + return unless(strings.HasPrefix(name, ".."), "it does not begin with two dots") + case "leading_dash": + return unless(strings.HasPrefix(name, "-"), "it does not begin with a dash") + case "leading_ideographic_space": + return unless(strings.HasPrefix(name, string(rune(0x3000))), "it does not begin with an ideographic space") + case "unicode_tags": + return unless(untagged(stem) == "hidden note", "it carries no hidden note in tag characters") + } + if r, ok := marked[target]; ok && !strings.ContainsRune(name, r) { + return "it lacks the character it is about" + } + return "" +} + +func unless(ok bool, fault string) string { + if ok { + return "" + } + return fault +} + +// untagged is the text the tag characters of s spell. +func untagged(s string) string { + var b strings.Builder + for _, r := range s { + if r >= 0xE0020 && r <= 0xE007E { + b.WriteRune(r - 0xE0000) + } + } + return b.String() +} + +// The set promises acceptance only where refusing would be the system's +// fault, and says unspecified everywhere else (MF5), with a reason the +// manifest already has - the owner's decision of 2026-09-24. +func TestTheFileNamePresetPromisesOnlyWhatASystemMustDo(t *testing.T) { + expanded, err := preset.Expand("filename-handling", preset.Args{}) + if err != nil { + t.Fatal(err) + } + rec, err := recipe.Parse(expanded.Source, "filename-handling") + if err != nil { + t.Fatal(err) + } + var accepted []string + for _, target := range rec.Targets { + switch target.Expected { + case "accept": + accepted = append(accepted, target.ID) + case "unspecified": + switch target.ExpectedReason { + case "filename_invalid", "filename_too_long", "filename_traversal": + default: + t.Errorf("%s is unspecified for %q, which is not one of the three name reasons", target.ID, target.ExpectedReason) + } + default: + t.Errorf("%s expects %q, and a name is either accepted or left to the system's policy", target.ID, target.Expected) + } + } + sort.Strings(accepted) + if got := strings.Join(accepted, " "); got != "leading_zeros many_dots null_word upper_extension" { + t.Errorf("the set promises acceptance for %q", got) + } +} + +// Every name is written as it was asked for, here - and CI runs this on +// Windows, Linux and macOS, which is the measurement of +// docs/NAMES-PRESET-2026-09-24.md section 4 kept. +// +// Asked in the default format and in one whose extension is a byte longer, +// because the names that are about length are made to a length with the +// extension, and only the default was ever measured by hand. +func TestTheFileNamePresetWritesEveryNameByteForByte(t *testing.T) { + for _, args := range [][]string{nil, {"--format", "docx"}} { + dir := t.TempDir() + var out, errOut bytes.Buffer + cmd := append([]string{"generate", "--preset", "filename-handling", "--out", dir}, args...) + if code := cli.Run(context.Background(), cmd, &out, &errOut); code != cli.ExitOK { + t.Fatalf("%v ended %d:\n%s", cmd, code, errOut.String()) + } + m, err := manifest.Load(filepath.Join(dir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + var recorded []string + for _, f := range m.Files { + if !f.Materialized { + t.Errorf("%v: %+q was not written", args, f.Name) + } + recorded = append(recorded, f.Name) + } + sort.Strings(recorded) + var onDisk []string + for _, name := range namesIn(t, dir) { + if name != "manifest.json" { + onDisk = append(onDisk, name) + } + } + if len(recorded) != 50 || strings.Join(onDisk, "\x00") != strings.Join(recorded, "\x00") { + t.Errorf("%v: the directory holds %d names and the manifest %d, and they are not the same bytes:\n disk %+q\n manifest %+q", + args, len(onDisk), len(recorded), onDisk, recorded) + } + if code := cli.Run(context.Background(), []string{"verify", filepath.Join(dir, "manifest.json")}, &out, &errOut); code != cli.ExitOK { + t.Errorf("%v: verify ended %d:\n%s", args, code, errOut.String()) + } + } +} + +// One preset, both surfaces, the same files - the names are the point, so the +// window has to write the same fifty. +func TestThePresetScreenWritesTheNamesTheCommandLineWrites(t *testing.T) { + fromCLI, fromWindow := t.TempDir(), t.TempDir() + var out, errOut bytes.Buffer + if code := cli.Run(context.Background(), []string{"generate", "--preset", "filename-handling", "--out", fromCLI}, &out, &errOut); code != cli.ExitOK { + t.Fatalf("the command line refused the preset: exit %d\n%s", code, errOut.String()) + } + + host, content := presetScreen(t) + choosePreset(t, content, "filename-handling") + fill(t, content, text.FieldOutputDir(), fromWindow) + press(t, content, "Generate") + waitForManifest(t, host, fromWindow) + join(host) + + cliNames, windowNames := namesIn(t, fromCLI), namesIn(t, fromWindow) + if len(cliNames) != 51 { + t.Fatalf("the command line wrote %d things and fifty files and a manifest were expected", len(cliNames)) + } + if strings.Join(cliNames, "\x00") != strings.Join(windowNames, "\x00") { + t.Fatalf("the two surfaces wrote different names:\n command line %+q\n window %+q", cliNames, windowNames) + } + for _, name := range cliNames { + if name == "manifest.json" { + continue + } + a, errA := os.ReadFile(filepath.Join(fromCLI, name)) + b, errB := os.ReadFile(filepath.Join(fromWindow, name)) + if errA != nil || errB != nil || !bytes.Equal(a, b) { + t.Errorf("%+q differs between the surfaces (%v, %v)", name, errA, errB) + } + } +} diff --git a/internal/guard/hiddencharacters_test.go b/internal/guard/hiddencharacters_test.go new file mode 100644 index 00000000..7d54506f --- /dev/null +++ b/internal/guard/hiddencharacters_test.go @@ -0,0 +1,98 @@ +package guard + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" +) + +// No file in this repository carries a character nobody can see. +// +// The class a reviewer reading a diff cannot catch by reading: a right to left +// override that makes code say one thing on screen and another to the +// compiler - the attack published as Trojan Source - a zero width space that +// makes two identifiers look like one, a byte order mark or a line separator +// inside a string. The preset of unusual file names holds exactly these +// characters as data, so its source is the first place in this tree that +// wants them, and it writes every one of them as an escape. Measured before +// this guard existed, on 2026-09-24: not one tracked file carried such a +// character, so the list of exceptions starts empty. +// +// And the tool that writes this code is the reason it is a guard rather than a +// habit. The editor this project is written with turns a typed backslash-u +// escape into the character itself, silently - it did so twice on the day this +// guard was written, once in a document about this very problem. +// +// A file that is not UTF-8 is not text and is left out, counted. The tab, the +// line feed and the carriage return are what text is laid out with. +func TestNoTrackedFileCarriesACharacterNobodyCanSee(t *testing.T) { + root := repoRoot(t) + listed := strings.Split(gitOutput(t, "ls-files", "-z", "--cached", "--others", "--exclude-standard"), "\x00") + + read, goFiles, binary := 0, 0, 0 + var faults []string + for _, f := range listed { + if f == "" { + continue + } + body, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(f))) + if err != nil { + continue + } + if !utf8.Valid(body) { + binary++ + continue + } + read++ + if strings.HasSuffix(f, ".go") { + goFiles++ + } + if fault := firstHidden(string(body)); fault != "" { + faults = append(faults, f+":"+fault) + } + } + + // The state this is about: the Go source was actually read. A listing + // that came back empty or from the wrong directory would pass below. + if goFiles < 100 { + t.Fatalf("only %d Go files were read (%d text files, %d not UTF-8), so this guard is not looking at this repository", goFiles, read, binary) + } + if len(faults) > 0 { + t.Errorf("%d file(s) carry a character nobody can see. Write it as an escape instead - in Go, a backslash, u and four hex digits:\n %s", + len(faults), strings.Join(faults, "\n ")) + } + t.Logf("%d text files read, %d of them Go, %d not UTF-8 left out", read, goFiles, binary) +} + +// firstHidden is where the first such character of text is, as "line: U+XXXX", +// or "" when there is none. +func firstHidden(text string) string { + line := 1 + for _, r := range text { + switch { + case r == '\n': + line++ + case r == '\t' || r == '\r': + case unseenHere(r): + return fmt.Sprintf("%d: U+%04X", line, r) + } + } + return "" +} + +// The detector finds what it is for, or the guard above passes by finding +// nothing anywhere. Built from rune numbers, since this file is one of the +// files being scanned. +func TestTheHiddenCharacterDetectorFindsWhatItIsFor(t *testing.T) { + for _, r := range []rune{0x202E, 0x200B, 0xFEFF, 0x2028, 0x00A0, 0x3000, 0xE0041} { + if got := firstHidden("ok\nx := \"a" + string(r) + "b\"\n"); got != fmt.Sprintf("2: U+%04X", r) { + t.Errorf("U+%04X on the second line was reported as %q", r, got) + } + } + if got := firstHidden("tab\there, crlf\r\nand a plain space\n"); got != "" { + t.Errorf("ordinary layout was reported as %q", got) + } +} diff --git a/internal/guard/parity_test.go b/internal/guard/parity_test.go index 6471eafa..fc2f051e 100644 --- a/internal/guard/parity_test.go +++ b/internal/guard/parity_test.go @@ -87,6 +87,13 @@ var reachableFromTheWindow = []string{ "preset:upload-validation.far-over", "preset:upload-validation.bulk", + // The preset of unusual file names, 2026-09-24, and the global flag it + // gives txt to. Run from the screen by + // TestThePresetScreenWritesTheNamesTheCommandLineWrites, and the menu's + // first value asked by TestEveryFlagAPresetReadsDefaultsToOneValueOnBothSurfaces. + "preset:filename-handling", + "preset:filename-handling.format", + // A recipe that builds on a preset, since 2026-09-22: the switch and the // menu on the batch screen are the extends key, and the chosen preset's // parameters under it are the with section, drawn from the declaration diff --git a/internal/guard/presetbytes_test.go b/internal/guard/presetbytes_test.go index 1852b3e5..e51eb70d 100644 --- a/internal/guard/presetbytes_test.go +++ b/internal/guard/presetbytes_test.go @@ -67,6 +67,11 @@ func TestEjectingAPresetGivesTheBytesItAlwaysGave(t *testing.T) { {id: "text-encoding", args: preset.Args{"sample": "8kb"}, bytes: 4570, sum: "f87c73864e5f517abb08b50393cd9a1681a90a30560c1d14dfddcf31e8037479"}, {id: "empty-and-minimal", args: preset.Args{}, bytes: 3811, sum: "80641962ac9dfb303f812fd78e0a0d1080f714094d159d7b10448291debb9279"}, {id: "empty-and-minimal", args: preset.Args{"formats": "jpg,png,txt"}, bytes: 743, sum: "4fd23e4b06a2e27ede987ab48a2cc302cc2accb9f948c7d5d25f675681f31f69"}, + // The preset of unusual file names, measured 2026-09-25 on its first + // build: the default, and a format whose extension is a byte longer, + // since the names about length are made to a length with it. + {id: "filename-handling", args: preset.Args{}, bytes: 10324, sum: "fc051b2285c0efed30bc5e19920e6f08e25fc8f95e3b1be0ed8d228a25310e43"}, + {id: "filename-handling", args: preset.Args{"format": "docx"}, bytes: 10418, sum: "65fcae1471cd3b0111cae3dba14cd7d3d39beb6f998dc665b3dbaec898aade86"}, } for _, want := range pinned { diff --git a/internal/guard/readdefaults_test.go b/internal/guard/readdefaults_test.go new file mode 100644 index 00000000..caac2fe6 --- /dev/null +++ b/internal/guard/readdefaults_test.go @@ -0,0 +1,81 @@ +package guard + +import ( + "bytes" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/gui/parts" + "github.com/donislawdev/TestingFilesGenerator/internal/gui/text" + "github.com/donislawdev/TestingFilesGenerator/internal/preset" +) + +// A flag a preset reads stands in with one value, whichever surface asks. +// +// Until 2026-09-24 the window took the value from preset.Global, which knew +// one default for --format - pdf, the one size-boundaries uses - while each +// preset's Expand applied its own. The second preset to read --format was the +// preset of unusual file names, made of text files by default, so the command +// line would have written text files and the window PDFs from one preset (D1). +// The default is declared by the preset now, and this asks both surfaces for +// it: the set a run makes with the flag left out, and the value the window's +// menu opens on. +func TestEveryFlagAPresetReadsDefaultsToOneValueOnBothSurfaces(t *testing.T) { + _, content := presetScreen(t) + + checked := 0 + for _, p := range preset.All() { + for _, name := range p.Reads { + declared := p.ReadDefaults[name] + + bare, err := preset.Expand(p.ID, preset.Args{}) + if err != nil { + t.Fatalf("%s refused its own defaults: %v", p.ID, err) + } + stated, err := preset.Expand(p.ID, preset.Args{name: declared}) + if err != nil { + t.Fatalf("%s refused --%s %s: %v", p.ID, name, declared, err) + } + if !bytes.Equal(bare.Source, stated.Source) { + t.Errorf("%s declares --%s %s, and leaving the flag out makes a different set", p.ID, name, declared) + } + // The comparison above says nothing if the flag changes nothing, so + // another value has to make another set. + if !anotherValueChangesTheSet(t, p, name, declared, bare.Source) { + t.Errorf("%s makes the same set whatever --%s says, so this guard cannot tell the defaults apart", p.ID, name) + } + + choosePreset(t, content, p.ID) + menu, ok := controlUnder(content, text.SettingLabel(name)).(*parts.Chooser) + if !ok { + t.Errorf("the window draws no menu for --%s of %s", name, p.ID) + continue + } + if menu.Selected != declared { + t.Errorf("the window opens --%s of %s on %q, and the command line uses %q", name, p.ID, menu.Selected, declared) + } + checked++ + } + } + if checked == 0 { + t.Fatal("no preset reads a flag, so this guard checked nothing") + } + t.Logf("%d flag(s) read by a preset, each defaulting to one value on both surfaces", checked) +} + +// anotherValueChangesTheSet is whether some other format makes a different +// set - the first one the preset accepts. +func anotherValueChangesTheSet(t *testing.T, p preset.Preset, name, declared string, bare []byte) bool { + t.Helper() + for _, id := range format.IDs() { + if id == declared { + continue + } + other, err := preset.Expand(p.ID, preset.Args{name: id}) + if err != nil { + continue + } + return !bytes.Equal(other.Source, bare) + } + return false +} diff --git a/internal/guard/testdata/screens/preset-menu.png b/internal/guard/testdata/screens/preset-menu.png index 3f7a4d75..68ad1acc 100644 Binary files a/internal/guard/testdata/screens/preset-menu.png and b/internal/guard/testdata/screens/preset-menu.png differ diff --git a/internal/guard/testdata/screens/preset-menu.xml b/internal/guard/testdata/screens/preset-menu.xml index 197581d7..682b3b60 100644 --- a/internal/guard/testdata/screens/preset-menu.xml +++ b/internal/guard/testdata/screens/preset-menu.xml @@ -393,16 +393,16 @@ - - - - - - + + + + + + - - - + + + @@ -410,17 +410,21 @@ - size-boundaries + filename-handling - tabular-import + size-boundaries - text-encoding + tabular-import + + text-encoding + + upload-validation diff --git a/internal/guard/unicodespace_test.go b/internal/guard/unicodespace_test.go new file mode 100644 index 00000000..e1af8459 --- /dev/null +++ b/internal/guard/unicodespace_test.go @@ -0,0 +1,106 @@ +package guard + +import ( + "strings" + "testing" + "unicode" + + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// A space from outside ASCII at either end of a name is part of the name. +// +// Measured on 2026-09-24 (O243): a recipe asking for a file whose name starts +// with an ideographic space, U+3000, got a file without it, and nothing said +// so. The parser took each value with strings.TrimSpace, which knows every +// white space character Unicode has, while YAML itself counts only the space +// and the tab. The library handed the character over intact and the reading +// code threw it away. It surfaced through the preset of unusual file names, +// which asks for exactly such a name and came back one short on every format. +// +// The guard beside it in compose_test.go puts every character in the MIDDLE +// of a value, on purpose, because the ends are trimmed. That is why it never +// saw this, and why this one asks about nothing but the ends. +// +// The characters come from the Unicode table rather than from a list typed +// here, so this asks about every one the language knows and not the few +// somebody thought of. ASCII ones are left out: a space or a tab at the end of +// an unquoted YAML value is not part of it by the rules of YAML. +func TestAUnicodeSpaceAtEitherEndOfANameIsKept(t *testing.T) { + var spaces []rune + for _, r16 := range unicode.White_Space.R16 { + for r := rune(r16.Lo); r <= rune(r16.Hi); r += rune(r16.Stride) { + if r > unicode.MaxASCII { + spaces = append(spaces, r) + } + } + } + for _, r32 := range unicode.White_Space.R32 { + for r := rune(r32.Lo); r <= rune(r32.Hi); r += rune(r32.Stride) { + spaces = append(spaces, r) + } + } + // The ideographic space the defect was found with, and the no break space + // every keyboard layout can type, have to be among them, or this is asking + // about some other table. + if !containsRune(spaces, 0x3000) || !containsRune(spaces, 0xA0) { + t.Fatalf("the table gave %d characters and not the two this is about: %U", len(spaces), spaces) + } + + checked := 0 + for _, r := range spaces { + for _, name := range []string{string(r) + "report.txt", "report.txt" + string(r)} { + // The state this guard is about: a name that a reader trimming + // Unicode white space would shorten. Asserted, not assumed. + if strings.TrimSpace(name) == name { + t.Fatalf("%q is not a name that trimming would change, so it tests nothing", name) + } + + written := "version: 1\ntargets:\n - id: t\n format: txt\n size: 1kb\n name: " + name + "\n" + if got, ok := nameReadFrom(t, []byte(written)); ok && got != name { + t.Errorf("a recipe written by hand asked for %+q and read it as %+q", name, got) + } + + composed, err := recipe.Compose(recipe.Document{Targets: []recipe.TargetDraft{{ + ID: "t", Format: "txt", Size: "1kb", Name: name, + }}}) + if err != nil { + t.Errorf("composing a recipe with the name %+q was refused: %v", name, err) + continue + } + if got, ok := nameReadFrom(t, composed); ok && got != name { + t.Errorf("a composed recipe asked for %+q and read it as %+q\n%s", name, got, composed) + } + checked++ + } + } + if checked == 0 { + t.Fatal("no name was checked - this guard would pass without asking anything") + } + t.Logf("%d names, %d characters at the start and at the end", checked, len(spaces)) +} + +// nameReadFrom parses a one target recipe and gives back the name it asks for. +func nameReadFrom(t *testing.T, src []byte) (string, bool) { + t.Helper() + rec, err := recipe.Parse(src, "guard") + if err != nil { + t.Errorf("the recipe was refused: %v\n%s", err, src) + return "", false + } + if len(rec.Targets) != 1 { + t.Errorf("the recipe parsed to %d targets\n%s", len(rec.Targets), src) + return "", false + } + return rec.Targets[0].Name, true +} + +func containsRune(rs []rune, want rune) bool { + for _, r := range rs { + if r == want { + return true + } + } + return false +} diff --git a/internal/guard/unseen_test.go b/internal/guard/unseen_test.go new file mode 100644 index 00000000..74c36854 --- /dev/null +++ b/internal/guard/unseen_test.go @@ -0,0 +1,113 @@ +package guard + +import ( + "testing" + "unicode" + + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// unseenHere is a character a person reading a line cannot see, worked out +// from the Unicode tables rather than asked of the code under test: anything +// that is not a letter, a mark, a number, punctuation or a symbol, apart from +// the plain space. A guard that imported the class from the code would agree +// with it whatever it did. +func unseenHere(r rune) bool { + return r != ' ' && !unicode.In(r, unicode.L, unicode.M, unicode.N, unicode.P, unicode.S) +} + +// unseenSample is the characters these guards ask about: every format +// character, every separator and every control character above the ones a +// recipe refuses outright, from the tables, and one each from private use and +// from the unassigned range. +func unseenSample() []rune { + var out []rune + add := func(t *unicode.RangeTable) { + for _, r16 := range t.R16 { + for r := rune(r16.Lo); r <= rune(r16.Hi); r += rune(r16.Stride) { + out = append(out, r) + } + } + for _, r32 := range t.R32 { + for r := rune(r32.Lo); r <= rune(r32.Hi); r += rune(r32.Stride) { + out = append(out, r) + } + } + } + add(unicode.Cf) + add(unicode.Zl) + add(unicode.Zp) + add(unicode.Zs) + for r := rune(0x80); r <= 0x9F; r++ { + out = append(out, r) + } + out = append(out, 0xE000, 0x0378) + + kept := out[:0] + for _, r := range out { + if unseenHere(r) { + kept = append(kept, r) + } + } + return kept +} + +// A character nobody can see goes into a composed recipe as an escape and +// comes back out as itself. +// +// Measured on 2026-09-24 (O244): the library wrote a right to left override, a +// zero width space, a byte order mark and a line separator raw into unquoted +// values. A person editing the recipe could not see what was in it, and PyYAML +// refused the document at the line separator. The preset of unusual file names +// ejects exactly such a recipe, and a recipe composed in the window can hold one +// too. +// +// Both halves are asked, because either alone is satisfied by the wrong code: +// nothing raw in the source is what writing the value as an empty string would +// also give, and the value coming back is what writing it raw gave before. +func TestACharacterNobodyCanSeeIsWrittenAsAnEscapeAndReadBackAsItself(t *testing.T) { + sample := unseenSample() + // The ones the defect was found with have to be among them, or this is + // asking about some other table. + for _, must := range []rune{0x202E, 0x200B, 0xFEFF, 0x2028, 0x3000, 0xA0, 0xE0068} { + if !containsRune(sample, must) { + t.Fatalf("U+%04X is not in the sample of %d characters", must, len(sample)) + } + } + + checked := 0 + for _, r := range sample { + for _, value := range []string{string(r) + "b.txt", "a" + string(r) + "b.txt", "ab.txt" + string(r)} { + src, err := recipe.Compose(recipe.Document{Targets: []recipe.TargetDraft{{ + ID: "t", Format: "txt", Size: "1kb", Name: value, Group: value, + }}}) + if err != nil { + t.Errorf("U+%04X: composing was refused: %v", r, err) + continue + } + for _, c := range string(src) { + if c != '\n' && unseenHere(c) { + t.Errorf("U+%04X: the composed recipe carries U+%04X raw\n%s", r, c, src) + break + } + } + rec, err := recipe.Parse(src, "composed") + if err != nil { + t.Errorf("U+%04X: the composed recipe was refused: %v\n%s", r, err, src) + continue + } + if got := rec.Targets[0].Name; got != value { + t.Errorf("U+%04X: asked for the name %+q and read %+q\n%s", r, value, got, src) + } + if got := rec.Targets[0].Group; got != value { + t.Errorf("U+%04X: asked for the group %+q and read %+q\n%s", r, value, got, src) + } + checked++ + } + } + if checked < 3*100 { + t.Fatalf("only %d values were checked - the tables gave less than they should", checked) + } + t.Logf("%d characters, %d values composed and read back", len(sample), checked) +} diff --git a/internal/guard/unseenoutput_test.go b/internal/guard/unseenoutput_test.go new file mode 100644 index 00000000..8885c503 --- /dev/null +++ b/internal/guard/unseenoutput_test.go @@ -0,0 +1,336 @@ +package guard + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/cli" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" + "github.com/donislawdev/TestingFilesGenerator/internal/gui/text" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// What this tool prints about a name a person cannot read shows that name, +// rather than letting it act (O241). +// +// Measured on 2026-09-24: verify reported a missing file named "photo", a +// right to left override, "gpj.txt" - and the terminal drew it as +// "phototxt.jpg" - and an extra "in", a zero width space, "voice.txt" that +// printed as "invoice.txt". The report named files other than the ones on the +// disk, and two of them could not be told apart. The preset of unusual file +// names writes exactly these, so every report about its files would lie in +// the same way. +// +// Asked from outside, through the commands, because the defect is a place that +// prints a name without going through core.Shown, and a new place tomorrow is +// the one nobody will remember. Every guard here asks two things: nothing in +// the text is a character nobody can see, and the escape is there - since +// printing nothing at all would pass the first alone. + +var ( + rightToLeft = string(rune(0x202E)) + zeroWidth = string(rune(0x200B)) + lineSeparator = string(rune(0x2028)) +) + +// escapeOf is how a character is written once it is shown. +func escapeOf(s string) string { + q := strconv.QuoteRune([]rune(s)[0]) + return q[1 : len(q)-1] +} + +// saysNothingUnseen fails when said holds a character nobody can see, other +// than the line breaks that separate what it says. +func saysNothingUnseen(t *testing.T, what, said string) { + t.Helper() + for _, r := range said { + if r != '\n' && r != '\r' && unseenHere(r) { + t.Errorf("%s printed U+%04X raw:\n%s", what, r, said) + return + } + } +} + +// saysEscaped fails when said does not carry the name with its escape. +func saysEscaped(t *testing.T, what, said, stem, unseen, rest string) { + t.Helper() + if want := stem + escapeOf(unseen) + rest; !strings.Contains(said, want) { + t.Errorf("%s does not show %q:\n%s", what, want, said) + } +} + +// unseenRun writes the three names into a directory and gives back the +// directory and the manifest. +func unseenRun(t *testing.T) (string, string) { + t.Helper() + // The directory has such a character too, so every line naming it is asked + // as well - verify and cleanup printed it raw until a review of 2026-09-25. + dir := filepath.Join(t.TempDir(), "run"+zeroWidth) + var drafts []recipe.TargetDraft + for i, name := range []string{"photo" + rightToLeft + "gpj.txt", "in" + zeroWidth + "voice.txt", "report" + lineSeparator + "ERROR.txt"} { + drafts = append(drafts, recipe.TargetDraft{ID: "t" + strconv.Itoa(i), Format: "txt", Size: "1kb", Name: name}) + } + src, err := recipe.Compose(recipe.Document{Targets: drafts}) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "names.yaml") + if err := os.WriteFile(path, src, 0o600); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := cli.Run(context.Background(), []string{"generate", path, "--out", dir}, &out, &errOut); code != cli.ExitOK { + t.Fatalf("the run ended %d:\n%s", code, errOut.String()) + } + // Joined here rather than read off the "manifest:" line, which shows the + // directory's character as an escape and so is not a path any more. + if !regexp.MustCompile(`(?m)^manifest: `).MatchString(errOut.String()) { + t.Fatalf("the run did not say where its manifest is:\n%s", errOut.String()) + } + return dir, filepath.Join(dir, "manifest.json") +} + +// carriesExactly fails unless some string in a JSON report is want, byte for +// byte - the half a program reads, which must not be escaped. +func carriesExactly(t *testing.T, what string, report []byte, want string) { + t.Helper() + var v any + if err := json.Unmarshal(report, &v); err != nil { + t.Errorf("%s is not JSON: %v\n%s", what, err, report) + return + } + if !holdsString(v, func(s string) bool { return strings.HasSuffix(s, want) }) { + t.Errorf("%s does not carry %+q exactly:\n%s", what, want, report) + } +} + +// linesWith is the lines of said that hold word, so a name is looked for on +// the line that says what happened to it. +func linesWith(said, word string) string { + var out []string + for _, line := range strings.Split(said, "\n") { + if strings.Contains(line, word) { + out = append(out, line) + } + } + return strings.Join(out, "\n") +} + +func holdsString(v any, match func(string) bool) bool { + switch x := v.(type) { + case string: + return match(x) + case []any: + for _, e := range x { + if holdsString(e, match) { + return true + } + } + case map[string]any: + for _, e := range x { + if holdsString(e, match) { + return true + } + } + } + return false +} + +func TestANoteAboutAFileShowsANameNobodyCanReadAsAnEscape(t *testing.T) { + // A text file of one byte cannot carry its label, which is a note about + // the file - the same trigger as the guard of notes about one file. + one := runCLI(t, "generate", "--format", "txt", "--size", "1b", "--count", "1", + "--name", "photo"+rightToLeft+"gpj.txt", "--dry-run", "--out", t.TempDir()) + if !strings.Contains(one, "note:") { + t.Fatalf("the run printed no note, so this guard checked nothing:\n%s", one) + } + saysNothingUnseen(t, "a note about one file", one) + saysEscaped(t, "a note about one file", one, "note: photo", rightToLeft, "gpj.txt: ") + + many := runCLI(t, "generate", "--format", "txt", "--size", "1b", "--count", "3", + "--name", "in"+zeroWidth+"voice_{index:04}.txt", "--dry-run", "--out", t.TempDir()) + if !strings.Contains(many, "3 files:") { + t.Fatalf("the run printed no grouped note, so this guard checked nothing:\n%s", many) + } + saysNothingUnseen(t, "a note about three files", many) + saysEscaped(t, "a note about three files", many, "in", zeroWidth, "voice_0001.txt") +} + +func TestVerifyShowsANameNobodyCanReadAsAnEscape(t *testing.T) { + dir, manifestPath := unseenRun(t) + if err := os.Remove(filepath.Join(dir, "photo"+rightToLeft+"gpj.txt")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "extra"+zeroWidth+".txt"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + + var out, errOut bytes.Buffer + if code := cli.Run(context.Background(), []string{"verify", manifestPath}, &out, &errOut); code != cli.ExitVerify { + t.Fatalf("verify ended %d rather than with a mismatch:\n%s%s", code, out.String(), errOut.String()) + } + said := out.String() + errOut.String() + saysNothingUnseen(t, "verify", said) + saysEscaped(t, "verify, the missing file", said, "photo", rightToLeft, "gpj.txt") + saysEscaped(t, "verify, the extra file", said, "extra", zeroWidth, ".txt") + + out.Reset() + errOut.Reset() + cli.Run(context.Background(), []string{"verify", "--json", manifestPath}, &out, &errOut) + carriesExactly(t, "verify --json", []byte(strings.TrimSpace(out.String()+errOut.String())), "photo"+rightToLeft+"gpj.txt") +} + +func TestCleanupShowsANameNobodyCanReadAsAnEscape(t *testing.T) { + dir, manifestPath := unseenRun(t) + // Changed since it was written, so cleanup keeps it and says why. + if err := os.WriteFile(filepath.Join(dir, "in"+zeroWidth+"voice.txt"), []byte("changed"), 0o600); err != nil { + t.Fatal(err) + } + + preview := runCLI(t, "cleanup", manifestPath) + saysNothingUnseen(t, "cleanup", preview) + saysEscaped(t, "cleanup, a file it would remove", linesWith(preview, "remove"), "photo", rightToLeft, "gpj.txt") + saysEscaped(t, "cleanup, a file it would keep", linesWith(preview, "keep"), "in", zeroWidth, "voice.txt") + + removed := runCLI(t, "cleanup", "--yes", manifestPath) + saysNothingUnseen(t, "cleanup --yes", removed) + saysEscaped(t, "cleanup --yes, the file it kept", linesWith(removed, "kept"), "in", zeroWidth, "voice.txt") + if _, err := os.Stat(filepath.Join(dir, "in"+zeroWidth+"voice.txt")); err != nil { + t.Errorf("the changed file was not kept: %v", err) + } +} + +func TestARefusalShowsANameNobodyCanReadAsAnEscape(t *testing.T) { + // Two names that are one file on most systems, told apart only by case. + src, err := recipe.Compose(recipe.Document{Targets: []recipe.TargetDraft{ + {ID: "lower", Format: "txt", Size: "1kb", Name: "a" + zeroWidth + ".txt"}, + {ID: "upper", Format: "txt", Size: "1kb", Name: "A" + zeroWidth + ".txt"}, + }}) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "collide.yaml") + if err := os.WriteFile(path, src, 0o600); err != nil { + t.Fatal(err) + } + refused := runCLI(t, "validate", path) + saysNothingUnseen(t, "a refusal of two names that collide", refused) + saysEscaped(t, "a refusal of two names that collide", refused, "A", zeroWidth, ".txt") + + // A file already there under the name asked for. + dir := t.TempDir() + name := "photo" + rightToLeft + "gpj.txt" + if err := os.WriteFile(filepath.Join(dir, name), []byte("mine"), 0o600); err != nil { + t.Fatal(err) + } + taken := runCLI(t, "generate", "--format", "txt", "--size", "1kb", "--count", "1", "--name", name, "--out", dir) + if !strings.Contains(taken, "already exists") { + t.Fatalf("the run did not refuse the taken name, so this guard checked nothing:\n%s", taken) + } + saysNothingUnseen(t, "a refusal of a taken name", taken) + saysEscaped(t, "a refusal of a taken name", taken, "photo", rightToLeft, "gpj.txt already exists") +} + +// The rest of what a run says about a place: the files of a boundary set, a +// neighbouring run's files, and an output directory that cannot be used. +func TestTheLinesAboutARunShowANameNobodyCanReadAsAnEscape(t *testing.T) { + boundary := runCLI(t, "generate", "--format", "txt", "--boundary", "2kb", + "--name", "b"+rightToLeft+"_{index:04}.txt", "--dry-run", "--out", t.TempDir()) + if !strings.Contains(boundary, "boundary") { + t.Fatalf("the run printed no boundary set, so this guard checked nothing:\n%s", boundary) + } + saysNothingUnseen(t, "a boundary set", boundary) + saysEscaped(t, "a boundary set", linesWith(boundary, "0002"), "b", rightToLeft, "_0002.txt") + + // A second run beside the first, recording itself under its own name. + dir, firstManifest := unseenRun(t) + src, err := recipe.Compose(recipe.Document{Manifest: "record" + zeroWidth + ".json", Targets: []recipe.TargetDraft{ + {ID: "second", Format: "txt", Size: "1kb", Name: "second" + zeroWidth + ".txt"}, + }}) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "second.yaml") + if err := os.WriteFile(path, src, 0o600); err != nil { + t.Fatal(err) + } + second := runCLI(t, "generate", path, "--out", dir) + if !strings.Contains(second, "manifest:") { + t.Fatalf("the second run did not record itself:\n%s", second) + } + saysNothingUnseen(t, "the line naming a run's manifest", second) + saysEscaped(t, "the line naming a run's manifest", linesWith(second, "manifest:"), "record", zeroWidth, ".json") + + neighbour := runCLI(t, "verify", firstManifest) + if !strings.Contains(neighbour, "another run's record") { + t.Fatalf("verify said nothing about the other run, so this guard checked nothing:\n%s", neighbour) + } + saysNothingUnseen(t, "a note about another run", neighbour) + saysEscaped(t, "a note about another run, its file", linesWith(neighbour, "another run"), "second", zeroWidth, ".txt") + saysEscaped(t, "a note about another run, its record", linesWith(neighbour, "another run"), "record", zeroWidth, ".json") + + // An output directory that is a file, and one another run is holding. + parent := t.TempDir() + file := filepath.Join(parent, "out"+rightToLeft+"file") + if err := os.WriteFile(file, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + notADir := runCLI(t, "generate", "--format", "txt", "--size", "1kb", "--out", file) + saysNothingUnseen(t, "a refusal of an output directory that is a file", notADir) + saysEscaped(t, "a refusal of an output directory that is a file", notADir, "out", rightToLeft, "file is a file") + + held := filepath.Join(parent, "held"+zeroWidth+"dir") + if err := os.MkdirAll(held, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(held, ".tfg-run-lock"), nil, 0o600); err != nil { + t.Fatal(err) + } + busy := runCLI(t, "generate", "--format", "txt", "--size", "1kb", "--out", held) + if !strings.Contains(busy, "another run is already writing") { + t.Fatalf("the run did not refuse a held directory, so this guard checked nothing:\n%s", busy) + } + saysNothingUnseen(t, "a refusal of a held directory", busy) + saysEscaped(t, "a refusal of a held directory", busy, "held", zeroWidth, "dir, so this one") + + // A directory inside that file cannot be made, and the system's own error + // under ours names the path again - raw, until a review of 2026-09-25. + underAFile := runCLI(t, "generate", "--format", "txt", "--size", "1kb", "--out", filepath.Join(file, "sub")) + if !strings.Contains(underAFile, "cannot create the output directory") { + t.Fatalf("the run did not refuse a directory inside a file, so this guard checked nothing:\n%s", underAFile) + } + saysNothingUnseen(t, "a refusal of a directory inside a file", underAFile) +} + +// The window says the same about an output directory as the command line: a +// folder named with a character nobody can see is shown with the escape, +// under the box it is about and at the foot of the form. +func TestTheWindowShowsADirectoryNobodyCanReadAsAnEscape(t *testing.T) { + parent := t.TempDir() + file := filepath.Join(parent, "out"+rightToLeft+"file") + if err := os.WriteFile(file, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + for _, out := range []string{file, filepath.Join(file, "sub")} { + host, content := presetScreen(t) + choosePreset(t, content, "filename-handling") + fill(t, content, text.FieldOutputDir(), out) + press(t, content, "Generate") + join(host) + + said := everythingSaid(content) + // The box itself holds what was typed, raw, which is right for a box. + said = strings.ReplaceAll(said, out, "") + if !strings.Contains(said, "out"+escapeOf(rightToLeft)+"file") { + t.Fatalf("the window did not refuse %+q, or refused it without naming it:\n%s", out, said) + } + saysNothingUnseen(t, "the window's refusal of "+out, said) + } +} diff --git a/internal/gui/window/runrefuse.go b/internal/gui/window/runrefuse.go index 021729da..6c5db171 100644 --- a/internal/gui/window/runrefuse.go +++ b/internal/gui/window/runrefuse.go @@ -49,7 +49,7 @@ func (r *runner) refuse(err error) { } // About the run rather than about one box, or about a setting this // screen does not draw. The foot of the form is where those belong. - loose = append(loose, one.Error()) + loose = append(loose, core.ShownText(one.Error())) } if len(loose) > 0 { r.problem.Say(strings.Join(loose, "\n\n")) diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 5ddf54bc..cbbdce8b 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -467,9 +467,9 @@ func (n *noteGroups) add(detail, name string) { // to make the large case better. func (g *noteGroup) line(detail string) string { if g.count == 1 { - return fmt.Sprintf("%s: %s", g.first[0], detail) + return fmt.Sprintf("%s: %s", core.Shown(g.first[0]), detail) } - named := strings.Join(g.first, ", ") + named := strings.Join(core.ShownEach(g.first), ", ") if hidden := g.count - len(g.first); hidden > 0 { return fmt.Sprintf("%s: %s Named: %s. %s not named here.", core.Count(g.count, "file", "files"), detail, named, diff --git a/internal/preset/build.go b/internal/preset/build.go index b62b49ab..ac7e46c8 100644 --- a/internal/preset/build.go +++ b/internal/preset/build.go @@ -182,6 +182,18 @@ func (f setFile) refused() error { return err } +// sampleAtLeast is how big a file about its name or its insides is: the +// sample, or the format's own floor with the label where that is larger, so +// that a format with a high floor cannot turn such a file into a refusal about +// a size. Shared since 2026-09-24 - upload-validation takes four kilobytes +// (sampleFor), the preset of unusual file names one. +func sampleAtLeast(desc format.Descriptor, sample int64) int64 { + if floor := format.SmallestWithLabel(desc); floor > sample { + return floor + } + return sample +} + // request is what refused asks the format - one place, so that a set that // skips a question it has already asked keys it by the question itself. func (f setFile) request() format.Request { diff --git a/internal/preset/filenamehandling.go b/internal/preset/filenamehandling.go new file mode 100644 index 00000000..594d9cf6 --- /dev/null +++ b/internal/preset/filenamehandling.go @@ -0,0 +1,269 @@ +package preset + +import ( + "fmt" + "strings" + + "golang.org/x/text/unicode/norm" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +const ( + namesID = "filename-handling" + + // namesFormat is the format the set is made of when nobody says. The name + // is what this preset is about and the contents are not, so the default is + // the format with the least in it, and --format changes it. + namesFormat = "txt" + + // namesQuestion is announced by the preset AND written into the header of + // an ejected recipe. + namesQuestion = "Will my system store, show and give back a file name it did not expect?" + + // namesSample is how big each file is when its format allows it - small, + // since every file is about its name, and the owner's budget for the set + // is fifty files of a kilobyte. + namesSample = 1 << 10 +) + +func init() { + Register(Preset{ + ID: namesID, + Title: "File name handling", + Question: namesQuestion, + + Reads: []string{"format"}, + ReadDefaults: map[string]string{"format": namesFormat}, + + Requires: []string{"MVP"}, + Catches: []string{ + "a name that looks like a different one on screen, in a log or in a list", + "a name cut, trimmed or rewritten between upload and storage", + "a length limit counted in characters where the storage counts bytes", + }, + + Expand: expandFileNames, + }) +} + +// The fifty names, chosen and measured in docs/NAMES-PRESET-2026-09-24.md. Each +// one catches a class of fault a system that takes files in has been seen to +// have, none repeats a class another one catches, and every one of them is +// written byte for byte on Windows, Linux and macOS - so a file that does not +// arrive, or arrives renamed, is the system under test and not this tool. +// +// A character nobody can see is written here as an escape and never as +// itself, and a guard reads every file of this repository for that +// (TestNoTrackedFileCarriesACharacterNobodyCanSee). A right to left override +// typed into Go source is how code comes to say one thing on screen and +// another to the compiler. +// +// expected is accept for the four names a system would be wrong to refuse, +// and unspecified for the rest, because whether a system takes a name with a +// zero width space in it is its own policy and not a fault (MF5). Only reasons +// the manifest already knows are used - the owner's decision. +var nameCases = []nameCase{ + // How a name goes through bytes, encodings and normalisation. + {id: "polish_diacritics", group: "scripts", stem: "zażółć gęślą jaźń"}, + {id: "cyrillic", group: "scripts", stem: "Отчёт за квартал"}, + {id: "cjk", group: "scripts", stem: "報告書"}, + // Arabic letters with digits, written as escapes so that the line reads + // the same in any editor: an Arabic word, 2024 and v2. + {id: "rtl_with_digits", group: "scripts", stem: "\u062A\u0642\u0631\u064A\u0631 2024 v2"}, + {id: "emoji", group: "scripts", stem: "🎉"}, + {id: "emoji_zwj", group: "scripts", stem: "👨\u200D👩\u200D👧 family"}, + {id: "nfd", group: "scripts", stem: "café", decomposed: true}, + {id: "hangul_nfd", group: "scripts", stem: "한국어 보고서", decomposed: true}, + {id: "case_mapping", group: "scripts", stem: "İstanbul Straße"}, + {id: "fullwidth", group: "scripts", stem: "ｒｅｐｏｒｔ"}, + {id: "combining_stack", group: "scripts", stem: "z\u0335\u0321a\u0337l\u0338g\u0336o"}, + + // What is shown is not what is stored. + {id: "bidi_override", group: "lookalike", stem: "photo\u202Egpj"}, + {id: "zero_width", group: "lookalike", stem: "in\u200Bvoice"}, + {id: "no_break_space", group: "lookalike", stem: "annual\u00A0report"}, + {id: "homoglyph", group: "lookalike", stem: "p\u0430ypal"}, + {id: "leading_bom", group: "lookalike", stem: "\uFEFFreport"}, + {id: "line_separator", group: "lookalike", stem: "report\u2028ERROR admin logged in"}, + {id: "unicode_tags", group: "lookalike", stem: "report" + tagged("hidden note")}, + + // Where a name is split, trimmed or hidden. + {id: "leading_space", group: "spaces-and-dots", stem: " leading space"}, + {id: "leading_ideographic_space", group: "spaces-and-dots", stem: "\u3000report"}, + {id: "double_space", group: "spaces-and-dots", stem: "two spaces"}, + {id: "leading_dot", group: "spaces-and-dots", stem: ".hidden"}, + {id: "leading_double_dot", group: "spaces-and-dots", stem: "..report", reason: "filename_traversal"}, + {id: "only_extension", group: "spaces-and-dots"}, + {id: "many_dots", group: "spaces-and-dots", stem: "v1.2.3.final", accepted: true}, + {id: "no_extension", group: "spaces-and-dots", stem: "README", extension: noExtension}, + {id: "upper_extension", group: "spaces-and-dots", stem: "REPORT", extension: upperExtension, accepted: true}, + + // A name that means something to a shell, a query or an address. + {id: "leading_dash", group: "metacharacters", stem: "-rf"}, + {id: "shell_substitution", group: "metacharacters", stem: "$(id) `id`"}, + {id: "shell_separators", group: "metacharacters", stem: "a;b&c"}, + {id: "sql_quote", group: "metacharacters", stem: "'; DROP TABLE files; --"}, + {id: "script_quote", group: "metacharacters", stem: "'-alert(1)-'"}, + {id: "url_encoded_traversal", group: "metacharacters", stem: "..%2F..%2Fetc%2Fpasswd", reason: "filename_traversal"}, + {id: "fullwidth_traversal", group: "metacharacters", stem: "．．／．．／ｅｔｃ／ｐａｓｓｗｄ", reason: "filename_traversal"}, + {id: "encoded_control", group: "metacharacters", stem: "report%00%0D%0A"}, + {id: "fullwidth_extension", group: "metacharacters", stem: "report", extension: fullwidthExtension}, + {id: "url_specials", group: "metacharacters", stem: "100% a+b #1"}, + {id: "formula", group: "metacharacters", stem: "=1+1"}, + + // Names that mean something to a server or a desktop, whatever the file + // holds - .htaccess with a PDF inside is still .htaccess. + {id: "htaccess", group: "special-names", stem: ".htaccess", extension: noExtension}, + {id: "web_config", group: "special-names", stem: "web.config", extension: noExtension}, + {id: "dotenv", group: "special-names", stem: ".env", extension: noExtension}, + {id: "ds_store", group: "special-names", stem: ".DS_Store", extension: noExtension}, + {id: "desktop_ini", group: "special-names", stem: "desktop.ini", extension: noExtension}, + {id: "office_lock", group: "special-names", stem: "~$report"}, + + // A name somebody reads as a value. + {id: "null_word", group: "values", stem: "null", accepted: true}, + {id: "leading_zeros", group: "values", stem: "007", accepted: true}, + + // Characters, bytes and UTF-16 units are three different numbers. Each is + // made to a length in bytes together with the extension, so the set means + // the same thing whatever --format says: 101 is one past what an ustar + // archive keeps, 255 is the most a file system stores. + {id: "ustar_101", group: "length", fill: "u", bytes: 101, reason: "filename_too_long"}, + {id: "max_ascii", group: "length", fill: "a", bytes: 255, reason: "filename_too_long"}, + {id: "cjk_bytes", group: "length", fill: "日", bytes: 255, reason: "filename_too_long"}, + {id: "emoji_bytes", group: "length", fill: "🎉", bytes: 255, reason: "filename_too_long"}, +} + +// nameCase is one file of the set, described by how its name is made rather +// than by the name, because the extension comes from the format. +type nameCase struct { + id, group string + // stem is the name before the extension. + stem string + // decomposed writes the stem in normalisation form D - an accent as a + // separate character after its letter, a Korean syllable as its letters - + // which is how macOS hands names over and how the source here cannot show + // them apart from the composed form. + decomposed bool + extension extensionRule + // fill and bytes make a name of a length rather than of a spelling: as + // many copies of fill as fit in bytes together with the extension. + fill string + bytes int + // accepted marks a name a system would be wrong to refuse. The rest are + // unspecified, for reason, or for filename_invalid when reason is empty. + accepted bool + reason string +} + +// extensionRule is what follows the stem. +type extensionRule int + +const ( + // formatExtension is the extension of the format, as it is. + formatExtension extensionRule = iota + // noExtension is a name that is complete as it is. + noExtension + // upperExtension is the extension of the format in capitals. + upperExtension + // fullwidthExtension is the extension of the format in full width + // letters, the dots left as they are. + fullwidthExtension +) + +// name is the file name for a format whose extension is ext. +func (c nameCase) name(desc format.Descriptor) (string, error) { + ext := desc.Extension + stem := c.stem + if c.decomposed { + stem = norm.NFD.String(stem) + } + if c.fill != "" { + copies := (c.bytes - len(ext)) / len(c.fill) + if copies < 1 { + return "", &format.PropertyValueError{Format: namesID, Key: "format", Value: desc.ID, + Reason: fmt.Sprintf("its extension %s is too long for a name of %d bytes, which the file %s is about. Choose a format with a shorter extension", ext, c.bytes, c.id)} + } + stem = strings.Repeat(c.fill, copies) + } + suffix := ext + switch c.extension { + case formatExtension: + case noExtension: + suffix = "" + case upperExtension: + suffix = strings.ToUpper(ext) + case fullwidthExtension: + suffix = fullwidth(ext) + } + return stem + suffix, nil +} + +// expectation is the outcome and the reason this file carries. +func (c nameCase) expectation() (string, string) { + switch { + case c.accepted: + return "accept", "" + case c.reason != "": + return "unspecified", c.reason + } + return "unspecified", "filename_invalid" +} + +// fullwidth writes the letters and digits of s in their full width forms, +// which NFKC turns back into the ones they stand for. The dot stays, so +// report.ｔｘｔ still has a dot before its extension. +func fullwidth(s string) string { + var b strings.Builder + for _, r := range s { + if r > ' ' && r <= '~' && r != '.' { + r += 0xFF01 - '!' + } + b.WriteRune(r) + } + return b.String() +} + +// tagged is s in Unicode tag characters: the same letters, invisible, and read +// by a language model as text. The words are neutral on purpose. A classic +// attack carries an instruction, and an instruction here would be one for +// every tool that reviews this source. +func tagged(s string) string { + var b strings.Builder + for _, r := range s { + b.WriteRune(0xE0000 + r) + } + return b.String() +} + +func expandFileNames(args Args) ([]byte, error) { + formatID := namesFormat + if v := args["format"]; v != "" { + formatID = v + } + desc, err := format.Get(formatID) + if err != nil { + return nil, err + } + size := sampleAtLeast(desc, namesSample) + + files := make([]setFile, 0, len(nameCases)) + for _, c := range nameCases { + name, err := c.name(desc) + if err != nil { + return nil, err + } + expected, reason := c.expectation() + files = append(files, setFile{ + id: c.id, name: name, group: c.group, desc: desc, size: size, + expected: expected, reason: reason, + }) + } + // Every file asks the format the same question - one format, one size, the + // label on - so asking it once answers for all fifty (PR7). + if err := files[0].refused(); err != nil { + return nil, err + } + return plan{preset: namesID, question: namesQuestion, targets: draftsOf(files)}.source() +} diff --git a/internal/preset/preset.go b/internal/preset/preset.go index bffa17b4..64046bd8 100644 --- a/internal/preset/preset.go +++ b/internal/preset/preset.go @@ -57,6 +57,20 @@ type Preset struct { // that is not there. Reads []string + // ReadDefaults is the value this preset gives each flag in Reads when the + // caller leaves it out, keyed by the flag's name. Register refuses a name + // in Reads without one. + // + // Declared here since 2026-09-24, when a second preset came to read + // --format with a default of its own. Until then the window took the + // default from Global, which knew only the pdf of size-boundaries, so the + // preset of unusual file names would have made text files from the command + // line and PDFs from the window - one preset, two sets (D1). Expand applies + // the default itself, and + // TestEveryFlagAPresetReadsDefaultsToOneValueOnBothSurfaces holds the two + // together. + ReadDefaults map[string]string + // Landing marks the preset a surface opens on before anybody has chosen // one. Exactly one preset sets it, and Register refuses a second. // @@ -188,10 +202,8 @@ func (p Preset) Check(args Args) error { // is a property of the build and registration order between two packages is not // something to rely on. // -// The default is the one the presets in this package use. That is true today -// with one preset and it is a coupling rather than a design: a second preset -// reading --format with a different default has to turn this into something the -// preset declares, and the constant it points at is the one place to notice. +// No default here. Which value stands in when nobody gives one belongs to the +// preset reading the flag - ReadDefaults - and Globals puts it in. func Global(name string) (format.Property, bool) { switch name { case "format": @@ -199,7 +211,6 @@ func Global(name string) (format.Property, bool) { Name: "format", Kind: format.PropertyChoice, Choices: format.IDs(), - Default: defaultFormat, Detail: "What kind of file the whole set is made of.", }, true } @@ -216,6 +227,7 @@ func (p Preset) Globals() []format.Property { out := make([]format.Property, 0, len(p.Reads)) for _, name := range p.Reads { if declared, ok := Global(name); ok { + declared.Default = p.ReadDefaults[name] out = append(out, declared) } } @@ -385,6 +397,11 @@ func Register(p Preset) { } landing = p.ID } + for _, name := range p.Reads { + if p.ReadDefaults[name] == "" { + panic(fmt.Sprintf("preset: %s reads --%s and gives it no default", p.ID, name)) + } + } // A parameter IS a format.Property, so a closed set of values is put in the // same order here as it is over there. One rule for both, in the place each // declaration passes through exactly once. diff --git a/internal/preset/sizeboundaries.go b/internal/preset/sizeboundaries.go index 061d9348..e4672dd0 100644 --- a/internal/preset/sizeboundaries.go +++ b/internal/preset/sizeboundaries.go @@ -39,7 +39,8 @@ func init() { Detail: "How far either side of the limit to reach, as a list of sizes.", }, }, - Reads: []string{"format"}, + Reads: []string{"format"}, + ReadDefaults: map[string]string{"format": defaultFormat}, SaidWhenDefaulted: map[string]string{ "limit": "no limit was given, so this set is built around " + defaultLimitText + diff --git a/internal/preset/uploadset.go b/internal/preset/uploadset.go index 096ca9c3..912f1a51 100644 --- a/internal/preset/uploadset.go +++ b/internal/preset/uploadset.go @@ -427,10 +427,7 @@ func wouldReach(floor, size, limit int64) int64 { // answer: 35% of expanding upload-validation, measured 2026-09-23 // (docs/GUI-MEMORY-2026-09-23.md section 4j). func sampleFor(desc format.Descriptor) int64 { - if floor := format.SmallestWithLabel(desc); floor > uploadSample { - return floor - } - return uploadSample + return sampleAtLeast(desc, uploadSample) } // farOverFiles is the one file well past the limit, or none when it was turned diff --git a/internal/recipe/compose.go b/internal/recipe/compose.go index c06a0b3d..6f277444 100644 --- a/internal/recipe/compose.go +++ b/internal/recipe/compose.go @@ -3,6 +3,7 @@ package recipe import ( "fmt" "strconv" + "unicode/utf8" "github.com/goccy/go-yaml" @@ -111,12 +112,12 @@ func Compose(d Document) ([]byte, error) { doc := yaml.MapSlice{{Key: "version", Value: SchemaVersion}} if d.Seed != "" { - doc = append(doc, yaml.MapItem{Key: "seed", Value: d.Seed}) + doc = append(doc, yaml.MapItem{Key: "seed", Value: written(d.Seed)}) } // The preset before the targets, because that is the order the run // takes them in. if d.Extends != "" { - doc = append(doc, yaml.MapItem{Key: KeyExtends, Value: presetScheme + d.Extends}) + doc = append(doc, yaml.MapItem{Key: KeyExtends, Value: written(presetScheme + d.Extends)}) } if with := withSection(d); len(with) > 0 { doc = append(doc, yaml.MapItem{Key: KeyWith, Value: with}) @@ -160,7 +161,7 @@ func Compose(d Document) ([]byte, error) { func withSection(d Document) yaml.MapSlice { var with yaml.MapSlice for _, name := range sortedKeys(d.With) { - with = append(with, yaml.MapItem{Key: name, Value: d.With[name]}) + with = append(with, yaml.MapItem{Key: name, Value: written(d.With[name])}) } return with } @@ -168,10 +169,10 @@ func withSection(d Document) yaml.MapSlice { func outputSection(d Document) yaml.MapSlice { var out yaml.MapSlice if d.OutDir != "" { - out = append(out, yaml.MapItem{Key: "dir", Value: d.OutDir}) + out = append(out, yaml.MapItem{Key: "dir", Value: written(d.OutDir)}) } if d.Manifest != "" { - out = append(out, yaml.MapItem{Key: "manifest", Value: d.Manifest}) + out = append(out, yaml.MapItem{Key: "manifest", Value: written(d.Manifest)}) } return out } @@ -182,7 +183,7 @@ func targetEntry(t TargetDraft) yaml.MapSlice { entry := yaml.MapSlice{} add := func(key, value string) { if value != "" { - entry = append(entry, yaml.MapItem{Key: key, Value: value}) + entry = append(entry, yaml.MapItem{Key: key, Value: written(value)}) } } // The keys where a number belongs are written as a number. See bareNumber. @@ -210,7 +211,7 @@ func targetEntry(t TargetDraft) yaml.MapSlice { // hashed into the manifest. Two runs of one screen have to compose the // same bytes or recipe_hash would move on its own. for _, name := range sortedKeys(t.Properties) { - props = append(props, yaml.MapItem{Key: name, Value: t.Properties[name]}) + props = append(props, yaml.MapItem{Key: name, Value: written(t.Properties[name])}) } entry = append(entry, yaml.MapItem{Key: "properties", Value: props}) } @@ -219,7 +220,7 @@ func targetEntry(t TargetDraft) yaml.MapSlice { for _, c := range t.Contains { one := yaml.MapSlice{} if c.Format != "" { - one = append(one, yaml.MapItem{Key: "format", Value: c.Format}) + one = append(one, yaml.MapItem{Key: "format", Value: written(c.Format)}) } if c.Count != "" { one = append(one, yaml.MapItem{Key: "count", Value: bareNumber(c.Count)}) @@ -254,11 +255,53 @@ func targetEntry(t TargetDraft) yaml.MapSlice { func bareNumber(value string) any { n, err := strconv.ParseInt(value, 10, 64) if err != nil || n < 0 || value != strconv.FormatInt(n, 10) { - return value + return written(value) } return n } +// written is a value the way the document carries it: as itself, unless it +// holds a character nobody reading the file can see. +// +// Such a value is written in double quotes with that character as an escape, +// and everything else in it as it is. Measured on 2026-09-24 (O244), when the +// preset of unusual file names put a right to left override, a zero width +// space, a byte order mark and a line separator into a recipe: the library +// wrote every one of them raw into an unquoted value. A person editing that +// recipe - which is what the header of an ejected one invites - could not see +// what they were editing, and a YAML 1.1 reader (PyYAML 6.0.3) refused the +// whole document at the line separator, which it takes for a line break. +// +// A value with nothing of the kind is written exactly as before, so no recipe +// this tool composed until then moves by a byte. +func written(value string) any { + if !core.HoldsUnseen(value) || !utf8.ValidString(value) { + return value + } + return escapedText(value) +} + +// escapedText is a value that goes into the document in double quotes, with +// escapes. +type escapedText string + +// MarshalYAML writes the value the way Go quotes a string, and that is YAML's +// double quoted form for everything that can reach here. The library takes +// the bytes as they are rather than choosing a style of its own - measured on +// 2026-09-24 on ten names, each read back as itself by the library, by Parse +// and by PyYAML. +// +// Go writes a quote and a backslash with a backslash in front, a character it +// cannot print as a backslash, a u and four hex digits, or a capital U and +// eight past the first plane, and a control character by its short name or as +// a backslash, an x and two hex digits. YAML reads every one of those the same +// way. The one form where they part is a byte that is not UTF-8, which Go +// writes as an x escape and YAML would read as a character - and written sends +// such a value on unchanged rather than here. +func (e escapedText) MarshalYAML() ([]byte, error) { + return []byte(strconv.Quote(string(e))), nil +} + // expectationEntry writes the short form when there is no reason and the long // one when there is, which is the same choice a person writing the file by hand // makes. Nil when nothing was stated. @@ -271,11 +314,11 @@ func expectationEntry(t TargetDraft) any { case t.Expected == "" && t.ExpectedReason == "": return nil case t.ExpectedReason == "": - return t.Expected + return written(t.Expected) default: return yaml.MapSlice{ - {Key: "outcome", Value: t.Expected}, - {Key: "reason", Value: t.ExpectedReason}, + {Key: "outcome", Value: written(t.Expected)}, + {Key: "reason", Value: written(t.ExpectedReason)}, } } } diff --git a/internal/recipe/scalar.go b/internal/recipe/scalar.go index d94bf8a0..759fbfc1 100644 --- a/internal/recipe/scalar.go +++ b/internal/recipe/scalar.go @@ -47,6 +47,18 @@ type scalar struct { quoted bool } +// yamlBlank is what may stand around a value without being part of it: the +// space and the tab YAML counts as white, and the line break the rendered node +// ends with. +// +// Not strings.TrimSpace, which was here until 2026-09-24 and knows every white +// space character Unicode has. A name beginning with an ideographic space or a +// no break space lost it on the way in, and the run wrote a file under a +// different name than the recipe asked for without a word (O243). The library +// had handed the character over intact. A space from outside ASCII at the end +// of a value is part of the value in YAML, so it is part of it here. +const yamlBlank = " \t\r\n" + // UnmarshalYAML takes the node as it was written. // // The source text of the node is the whole reason this type works. Measured @@ -71,7 +83,7 @@ type scalar struct { // instead of the node would have shown. func (s *scalar) UnmarshalYAML(n ast.Node) error { b := []byte(n.String()) - t := strings.TrimSpace(string(b)) + t := strings.Trim(string(b), yamlBlank) if len(t) >= 2 { first, last := t[0], t[len(t)-1] if (first == '"' && last == '"') || (first == '\'' && last == '\'') { diff --git a/web/content/en/site.json b/web/content/en/site.json index 0c10a354..3e52c382 100644 --- a/web/content/en/site.json +++ b/web/content/en/site.json @@ -97,6 +97,7 @@ }, "presets": { "empty-and-minimal": "Does a file that is valid and as small as the format allows get through?", + "filename-handling": "Will my system store, show and give back a file name it did not expect?", "size-boundaries": "Is a size limit enforced exactly where it is declared?", "tabular-import": "Does my table import survive what real tools export?", "text-encoding": "Does my reader know which encoding a file is in, or is it guessing?", diff --git a/web/content/pl/site.json b/web/content/pl/site.json index e63529fd..98a099ff 100644 --- a/web/content/pl/site.json +++ b/web/content/pl/site.json @@ -97,6 +97,7 @@ }, "presets": { "empty-and-minimal": "Czy plik poprawny i najmniejszy, na jaki format pozwala, przechodzi?", + "filename-handling": "Czy mój system zapisze, pokaże i odda nazwę pliku, której się nie spodziewał?", "size-boundaries": "Czy limit rozmiaru działa dokładnie tam, gdzie jest zadeklarowany?", "tabular-import": "Czy import tabeli poradzi sobie z tym, co eksportują prawdziwe narzędzia?", "text-encoding": "Czy mój czytnik wie, w jakim kodowaniu jest plik, czy zgaduje?", diff --git a/web/public/docs/index.html b/web/public/docs/index.html index b5716b89..c74cae05 100644 --- a/web/public/docs/index.html +++ b/web/public/docs/index.html @@ -311,6 +311,10 @@

What is a preset?

empty-and-minimal

Does a file that is valid and as small as the format allows get through?

+
  • +

    filename-handling

    +

    Will my system store, show and give back a file name it did not expect?

    +

  • size-boundaries

    Is a size limit enforced exactly where it is declared?

    diff --git a/web/public/pl/dokumentacja/index.html b/web/public/pl/dokumentacja/index.html index 61922942..6361aa59 100644 --- a/web/public/pl/dokumentacja/index.html +++ b/web/public/pl/dokumentacja/index.html @@ -312,6 +312,10 @@

    Czym jest preset?

    empty-and-minimal

    Czy plik poprawny i najmniejszy, na jaki format pozwala, przechodzi?

    • +
  • +

    filename-handling

    +

    Czy mój system zapisze, pokaże i odda nazwę pliku, której się nie spodziewał?

    +

  • size-boundaries

    Czy limit rozmiaru działa dokładnie tam, gdzie jest zadeklarowany?