diff --git a/README.md b/README.md index df3503f..3063237 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Relational Operator Replacement (ROR) — mutating `>`, `>=`, `<`, `<=`, `==`, ` - **`-threshold`** — CI quality gate (e.g., `-threshold 80` fails if score < 80%) - **`-json`** — Machine-readable output for CI pipelines (`-json -v` for NDJSON streaming) - **`-dry-run`** — Preview mutations without running tests -- **Low noise** — Automatically skips equivalent mutants (`len(x) > 0`) and simple error propagation (`if err != nil { return err }`) +- **Low noise** — Automatically skips equivalent local integer clamps (`if x < 1 { x = 1 }`) and simple error propagation (`if err != nil { return err }`) - **Zero dependencies** — Go standard library only --- @@ -153,28 +153,25 @@ When mutest reports a survivor like this: It means mutest swapped the operator and **no test noticed**: ```go -func Max(a, b int) int { - if a > b { // ← mutest changed this to >=, tests still passed - return a - } - return b +func IsAdult(age int) bool { + return age > 17 // ← mutest changed this to >=, tests still passed } ``` -The fix — add a test at the boundary: +Add a test at the boundary: ```go -func TestMax_EqualValues(t *testing.T) { - // This test kills the > → >= mutation because Max(3,3) - // returns 3 with >, but would return a (also 3) with >=. - // More importantly, it verifies the boundary behavior is intentional. - if got := Max(3, 3); got != 3 { - t.Errorf("Max(3,3) = %d, want 3", got) +func TestIsAdult_Boundary(t *testing.T) { + if IsAdult(17) { + t.Fatal("17 is below the adult age threshold") } } ``` Re-run mutest and that mutation point will now show `--- KILLED:`. +An equivalent mutation cannot be killed by adding tests: for example, changing +`>` to `>=` in an integer `Max(a, b)` that returns either operand produces the +same result when `a == b`. Review such sites and use `//mutest:skip` when appropriate. ### Exit Codes @@ -201,7 +198,7 @@ Positional arguments are package patterns (default: `./...`), following the same | `-diff` | | Only mutate lines changed relative to this git ref (e.g., `origin/main`) | | `-dry-run` | `false` | Discover mutations without running tests | | `-run` | | Regexp to pass to `go test -run` | -| `-workers` | `NumCPU` | Max parallel test processes | +| `-workers` | `NumCPU` | Max concurrent package builds / test processes; mutants in one package run sequentially | | `-timeout` | `30s` | Per-mutant test timeout | | `-threshold` | `0` | Minimum kill rate % (0-100); exit 1 if below. 0 = any survived fails | | `-skip-err-propagation` | `true` | Skip simple error propagation patterns (`if err != nil { return err }`) | @@ -367,9 +364,11 @@ $ mutest -dry-run -json ./... 3. **Instrument** — Replace each ordered comparison with a generic helper call (e.g., `a > b` → `_mutest_cmp_1(a, b)`) and each equality comparison with a flip of its result (`a == b` → `(a == b) != _mutest_on(1)`; the original comparison stays in place because its operands may legally have different static types, e.g. `any == error`), then generate a runtime file that switches behavior based on `MUTEST_ID` 4. **Build** — Compile one test binary per package with all mutations embedded 5. **Verify baseline** — Run each test binary once with **no** mutation active. If any package's tests fail without a mutation, mutest aborts (a broken or flaky suite would otherwise make every mutant a false KILLED) -6. **Test** — Run the pre-built binary once per mutation with `MUTEST_ID=N`, in a parallel worker pool. Each binary runs with its package directory as the working directory, so `testdata` relative paths resolve +6. **Test** — Run the pre-built binary once per mutation with `MUTEST_ID=N`, in a worker pool that runs packages in parallel and mutants within a package sequentially to avoid shared fixture conflicts. Each binary runs with its package directory as the working directory, so `testdata` relative paths resolve 7. **Judge** — `exit 0` = survived (test gap), `exit != 0` = killed (caught), timeout = detected (hung) +Thresholds use the unrounded kill rate. Captured build/test output retains at most the last 64 KiB per process, with a truncation marker when needed. Timeouts cancel the test process tree on Unix and Windows. + A package with **no test files** builds no binary; its mutants are reported as `SURVIVED` (no tests is the largest possible test gap). If the run is interrupted (e.g. `Ctrl+C`), untested mutants are reported as `CANCELED` and mutest exits non-zero rather than judging success on partial results. This **runtime mutation selection** approach compiles each package only once regardless of how many mutations it contains. Traditional per-mutation compilation (`N mutations × compile`) is replaced with `P packages × compile + N mutations × run`, dramatically reducing overhead. **Original source files are never modified.** @@ -380,13 +379,12 @@ mutest automatically skips mutations that are known to produce false positives: | Pattern | Example | Why skipped | |---------|---------|-------------| -| `len(x)` compared to `0` | `len(s) > 0` → `len(s) >= 0` | `len()` never returns negative, so the mutation cannot change behavior | -| `cap(x)` compared to `0` | `cap(s) > 0` → `cap(s) >= 0` | Same as `len()` — `cap()` is always non-negative | +| Local integer clamp | `if x < 1 { x = 1 }` | At equality, assigning the same constant leaves the local integer unchanged; no `else` branch | | Simple error propagation | `if err != nil { return err }` | Go's idiomatic error propagation — mutating these generates noise without meaningful test gaps | -Comparisons with non-zero literals (e.g., `len(s) > 1`) are **not** skipped — the boundary between 1 and 2 is meaningful. +Comparisons involving `len` or `cap`, including comparisons to zero, are tested: changing `len(s) > 0` to `len(s) >= 0` changes behavior for an empty slice. Clamp skipping excludes floats, map entries, globals, and expressions with side effects. -Complex error handling is **not** skipped — compound conditions (`err != nil && !timedOut`), fallback assignments, and multi-statement bodies represent real logic that should be tested. +Only direct propagation of an error-typed value (or a `fmt.Errorf` wrapper) under `err != nil` is skipped. Success branches (`err == nil`) are tested. Complex error handling is **not** skipped — compound conditions (`err != nil && !timedOut`), fallback assignments, and multi-statement bodies represent real logic that should be tested. To disable error propagation skipping and mutate all `err != nil` checks: diff --git a/cmd/mutest/regression_test.go b/cmd/mutest/regression_test.go new file mode 100644 index 0000000..3500027 --- /dev/null +++ b/cmd/mutest/regression_test.go @@ -0,0 +1,273 @@ +package mutest_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + mutest "github.com/fchimpan/mutest/cmd/mutest" + "github.com/fchimpan/mutest/output" +) + +func TestRegressionMutationSemantics(t *testing.T) { + cases := []struct { + name string + src string + test string + total int + killed int + }{ + { + name: "defined bool", + src: "type Flag bool\nfunc F(n int) Flag { return n > 0 }", + test: `if F(0) {t.Fatal("boundary")}`, + total: 1, + killed: 1, + }, + { + name: "constant precision", + src: "func F() bool {return 9007199254740992.0 < 9007199254740993.0}", + test: `if !F() {t.Fatal("precision")}`, + total: 1, + }, + { + name: "large constant", + src: "func F() bool {return 1<<100 > 0}", + test: `if !F() {t.Fatal("overflow")}`, + total: 1, + }, + { + name: "constant boundary", + src: "func F() bool {return 1<<100 <= 1<<100}", + test: `if !F() {t.Fatal("boundary")}`, + total: 1, + killed: 1, + }, + { + name: "len boundary", + src: "func F(s []int) bool {return len(s)>0}", + test: `if F(nil) {t.Fatal("empty")}`, + total: 1, + killed: 1, + }, + { + name: "fallback", + src: `func F(err error) string {if err!=nil {return "fallback"};return "ok"}`, + test: `if F(nil)!="ok" {t.Fail()}`, + total: 1, + killed: 1, + }, + { + name: "success branch", + src: `func F(err error) string {if err==nil {return "ok"};return "bad"}`, + test: `if F(nil)!="ok" {t.Fail()}`, + total: 1, + killed: 1, + }, + { + name: "direct propagation", + src: "func F(err error) error {if err!=nil {return err};return nil}", + test: `if F(nil)!=nil {t.Fail()}`, + }, + { + name: "wrapped propagation", + src: `import "fmt" +func F(err error) error {if err!=nil {return fmt.Errorf("wrap: %w",err)};return nil}`, + test: `if F(nil)!=nil {t.Fail()}`, + }, + { + name: "return side effects", + src: `var Calls int +func record()int{Calls++;return 0} +func F(err error)(int,error) {if err!=nil {return record(),err};return 0,nil}`, + test: `_,_=F(nil);if Calls!=0 {t.Fail()}`, + total: 1, + killed: 1, + }, + { + name: "typed non error", + src: "func F(err *int) int {if err!=nil {return 1};return 0}", + test: `if F(nil)!=0 {t.Fail()}`, + total: 1, + killed: 1, + }, + { + name: "operand side effects", + src: `var Calls int +func next() int {Calls++;return Calls} +func F() bool {return next()0}`, + test: "if F(0) {t.Fail()}", + total: 1, + killed: 1, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + if tt.name == "cgo" { + if os.Getenv("CGO_ENABLED") == "0" { + t.Skip("cgo disabled") + } + if _, err := exec.LookPath("cc"); err != nil { + t.Skip("C compiler unavailable") + } + } + dir := t.TempDir() + writeFiles(t, dir, map[string]string{ + "go.mod": "module regression\n\ngo 1.24.0\n", + "lib.go": "package regression\n" + tt.src + "\n", + "lib_test.go": "package regression\nimport \"testing\"\nfunc TestF(t *testing.T){" + tt.test + "}\n", + }) + chdir(t, dir) + var out, diag bytes.Buffer + err := mutest.Run(context.Background(), []string{"-json", "-workers", "1", "./..."}, &out, &diag) + if err != nil && !errors.Is(err, mutest.ErrTestsFailed) { + t.Fatalf("%v\n%s", err, diag.String()) + } + var s output.JSONSummary + if err := json.Unmarshal(out.Bytes(), &s); err != nil { + t.Fatal(err) + } + if s.Total != tt.total || s.Killed != tt.killed || s.Errors != 0 || s.Canceled != 0 { + t.Fatalf("got %+v", s) + } + }) + } +} + +func TestRegressionDiscoveryErrorsAndPositions(t *testing.T) { + for _, tt := range []struct { + name string + src string + wantErr bool + }{ + { + name: "syntax error", + src: "package p\nfunc F(n int) bool {return n > }", + wantErr: true, + }, + { + name: "physical lines", + src: "package p\n//line input.go:1000\nfunc F(n int) bool {\n//mutest:skip\nreturn n > 0\n}", + }, + } { + t.Run(tt.name, func(t *testing.T) { + d := t.TempDir() + writeFiles(t, d, map[string]string{ + "go.mod": "module p\ngo 1.24.0\n", + "lib.go": tt.src, + }) + chdir(t, d) + var out bytes.Buffer + err := mutest.Run(context.Background(), []string{"-dry-run", "-json", "./..."}, &out, io.Discard) + if tt.wantErr { + if !errors.Is(err, mutest.ErrDiscovery) { + t.Fatalf("got %v", err) + } + } else if err != nil || strings.TrimSpace(out.String()) != "[]" { + t.Fatalf("%v %s", err, &out) + } + }) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, fmt.Errorf("disk full") } +func TestRunPropagatesOutputFailure(t *testing.T) { + d := t.TempDir() + writeFiles(t, d, map[string]string{ + "go.mod": "module p\ngo 1.24.0\n", + "lib.go": "package p\n", + }) + chdir(t, d) + err := mutest.Run(context.Background(), []string{"-dry-run", "-json"}, failingWriter{}, io.Discard) + if err == nil || !strings.Contains(err.Error(), "disk full") { + t.Fatalf("got %v", err) + } +} +func TestRunCanceledBeforeDiscovery(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := mutest.Run(ctx, []string{"-dry-run"}, io.Discard, io.Discard) + if !errors.Is(err, mutest.ErrInterrupted) { + t.Fatalf("got %v", err) + } + if strings.Count(err.Error(), mutest.ErrInterrupted.Error()) != 1 { + t.Fatalf("duplicated interruption diagnostic: %v", err) + } +} + +func TestRunSerializesSharedPackageFixtures(t *testing.T) { + dir := t.TempDir() + writeFiles(t, dir, map[string]string{ + "go.mod": "module fixture\ngo 1.24.0\n", + "lib.go": `package fixture + func A(n int) bool {return n>0} + func B(n int) bool {return n<0} + `, + "lib_test.go": `package fixture + import("os";"testing";"time") + func TestSharedFixture(t *testing.T) { + f,err:=os.OpenFile("fixture.lock",os.O_CREATE|os.O_EXCL|os.O_WRONLY,0600) + if err!=nil {t.Fatal(err)} + f.Close() + defer os.Remove("fixture.lock") + time.Sleep(100*time.Millisecond) + } + `, + }) + chdir(t, dir) + var out, diag bytes.Buffer + err := mutest.Run(context.Background(), []string{"-json", "-workers", "2", "./..."}, &out, &diag) + if !errors.Is(err, mutest.ErrTestsFailed) { + t.Fatalf("%v: %s", err, &diag) + } + var s output.JSONSummary + if err := json.Unmarshal(out.Bytes(), &s); err != nil { + t.Fatal(err) + } + if s.Total != 2 || s.Survived != 2 || s.Killed != 0 { + t.Fatalf("false detection from fixture conflict: %+v", s) + } +} + +func writeFiles(t *testing.T, dir string, files map[string]string) { + t.Helper() + for name, content := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func chdir(t *testing.T, dir string) { + t.Helper() + original, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chdir(original); err != nil { + t.Errorf("restore working directory: %v", err) + } + }) +} diff --git a/cmd/mutest/run.go b/cmd/mutest/run.go index 1230435..a5feb24 100644 --- a/cmd/mutest/run.go +++ b/cmd/mutest/run.go @@ -90,11 +90,12 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if *showVersion { v, c, d := resolveVersion() if c != "none" { - fmt.Fprintf(stdout, "mutest %s (commit: %s, built: %s)\n", v, c, d) + _, err := fmt.Fprintf(stdout, "mutest %s (commit: %s, built: %s)\n", v, c, d) + return err } else { - fmt.Fprintf(stdout, "mutest %s\n", v) + _, err := fmt.Fprintf(stdout, "mutest %s\n", v) + return err } - return nil } patterns := fs.Args() @@ -129,7 +130,15 @@ func baselineErr(ctx context.Context, err error) error { return fmt.Errorf("%w: %w", ErrBaseline, err) } -func run(ctx context.Context, cfg config.Config, stdout, stderr io.Writer) error { +func run(ctx context.Context, cfg config.Config, stdout, stderr io.Writer) (retErr error) { + defer func() { + if ctx.Err() != nil && !errors.Is(retErr, ErrInterrupted) { + retErr = errors.Join(retErr, ErrInterrupted) + } + }() + if ctx.Err() != nil { + return ErrInterrupted + } if err := config.Validate(cfg); err != nil { return fmt.Errorf("%w: %w", ErrInvalidConfig, err) } @@ -138,7 +147,7 @@ func run(ctx context.Context, cfg config.Config, stdout, stderr io.Writer) error SkipErrPropagation: cfg.SkipErrPropagation, }) - points, err := eng.DiscoverAll() + points, err := eng.DiscoverAllContext(ctx) if err != nil { return fmt.Errorf("%w: %w", ErrDiscovery, err) } @@ -148,9 +157,14 @@ func run(ctx context.Context, cfg config.Config, stdout, stderr io.Writer) error return fmt.Errorf("failed to get current working directory: %w", err) } rep := output.NewReporter(cfg, stdout, stderr, cwd) + defer func() { + if err := rep.Err(); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("write output: %w", err)) + } + }() if cfg.Diff != "" { //mutest:skip - cl, err := diff.ParseGitDiff(cfg.Diff) + cl, err := diff.ParseGitDiffContext(ctx, cfg.Diff) if err != nil { return fmt.Errorf("%w: %w", ErrDiff, err) } @@ -176,25 +190,10 @@ func run(ctx context.Context, cfg config.Config, stdout, stderr io.Writer) error if err != nil { return fmt.Errorf("%w: %w", ErrInstrumentation, err) } - // Ensure temp dirs are cleaned up even on SIGINT/SIGTERM. - // defer alone is insufficient: os.Exit bypasses defers, and - // a killed process never runs them at all. - cleanupDone := make(chan struct{}) - go func() { - <-ctx.Done() - engine.CleanupInstrumented(pkgs) - close(cleanupDone) - }() - defer func() { - engine.CleanupInstrumented(pkgs) - select { - case <-cleanupDone: - default: - } - }() + defer engine.CleanupInstrumented(pkgs) fmt.Fprintf(rep.Info(), "mutest: building test binaries...\n") - if err := eng.BuildTestBinaries(ctx, pkgs); err != nil { + if err := eng.BuildTestBinaries(ctx, pkgs, cfg.Workers); err != nil { return fmt.Errorf("%w: %w", ErrBuild, err) } @@ -232,15 +231,19 @@ func run(ctx context.Context, cfg config.Config, stdout, stderr io.Writer) error return ErrInterrupted } - if cfg.Threshold > 0 { + return evaluateSummary(summary, cfg.Threshold) +} + +func evaluateSummary(summary *runner.Summary, threshold float64) error { + if summary.Canceled > 0 { + return ErrInterrupted + } + if threshold > 0 { // Errors mean mutants could not be tested; never silently pass them. if summary.Errors > 0 { return ErrTestsFailed } - // Compare against the same rounded value shown to the user (F11): a - // raw-rate comparison could fail "-threshold 80" on a run that - // displays "Score: 80.0%" whenever the unrounded rate was e.g. 79.96. - if output.RoundedKillRate(summary) < cfg.Threshold { + if output.CalcKillRate(summary) < threshold { return ErrTestsFailed } return nil diff --git a/cmd/mutest/run_test.go b/cmd/mutest/run_test.go index d5c8ccc..a6f6fbe 100644 --- a/cmd/mutest/run_test.go +++ b/cmd/mutest/run_test.go @@ -13,6 +13,7 @@ import ( "github.com/fchimpan/mutest/config" "github.com/fchimpan/mutest/output" + "github.com/fchimpan/mutest/runner" ) func chdir(t *testing.T, dir string) { @@ -522,6 +523,16 @@ func TestThreshold(t *testing.T) { } } +func TestEvaluateSummaryUsesExactRate(t *testing.T) { + s := &runner.Summary{Total: 2001, Killed: 2000, Survived: 1} + if !errors.Is(evaluateSummary(s, 100), ErrTestsFailed) { + t.Fatal("survivor passed threshold 100") + } + if evaluateSummary(s, 99.9) != nil { + t.Fatal("valid threshold rejected") + } +} + func TestRun_Threshold_Zero_DefaultBehavior(t *testing.T) { chdir(t, "../../testdata/project") diff --git a/config/config.go b/config/config.go index 4ba256d..a31d609 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" + "math" "time" ) @@ -28,7 +29,7 @@ func Validate(c Config) error { if c.Timeout <= 0 { return fmt.Errorf("-timeout must be > 0, got %s", c.Timeout) } - if c.Threshold < 0 || c.Threshold > 100 { + if math.IsNaN(c.Threshold) || math.IsInf(c.Threshold, 0) || c.Threshold < 0 || c.Threshold > 100 { return fmt.Errorf("-threshold must be between 0 and 100, got %.1f", c.Threshold) } return nil diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..eb47cb2 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,42 @@ +package config + +import ( + "math" + "testing" + "time" +) + +func TestRejectNonFiniteThreshold(t *testing.T) { + for _, v := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} { + if Validate(Config{Workers: 1, Timeout: time.Second, Threshold: v}) == nil { + t.Fatalf("accepted %v", v) + } + } +} + +func TestThresholdBoundaries(t *testing.T) { + for _, tt := range []struct { + threshold float64 + valid bool + }{ + { + threshold: 0, + valid: true, + }, + { + threshold: 100, + valid: true, + }, + { + threshold: math.Nextafter(0, -1), + }, + { + threshold: math.Nextafter(100, 101), + }, + } { + err := Validate(Config{Workers: 1, Timeout: time.Second, Threshold: tt.threshold}) + if (err == nil) != tt.valid { + t.Errorf("threshold %v: %v", tt.threshold, err) + } + } +} diff --git a/diff/git.go b/diff/git.go index 3dc3422..98da029 100644 --- a/diff/git.go +++ b/diff/git.go @@ -1,6 +1,7 @@ package diff import ( + "context" "fmt" "os" "os/exec" @@ -8,6 +9,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/fchimpan/mutest/internal/process" ) var hunkRe = regexp.MustCompile(`@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@`) @@ -21,12 +24,19 @@ var hunkRe = regexp.MustCompile(`@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@`) // //mutest:skip func ParseGitDiff(baseRef string) (ChangedLines, error) { - root, err := repoRoot() + return ParseGitDiffContext(context.Background(), baseRef) +} + +func ParseGitDiffContext(ctx context.Context, baseRef string) (ChangedLines, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + root, err := repoRootContext(ctx) if err != nil { return nil, fmt.Errorf("git repo root: %w", err) } - mbOut, err := exec.Command("git", "merge-base", baseRef, "HEAD").Output() + mbOut, err := process.Command(ctx, "git", "merge-base", "--", baseRef, "HEAD").Output() if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return nil, fmt.Errorf("git merge-base %s HEAD failed: %s", baseRef, strings.TrimSpace(string(exitErr.Stderr))) @@ -39,10 +49,12 @@ func ParseGitDiff(baseRef string) (ChangedLines, error) { // working tree, which is what mutation discovery parses. core.quotepath=off // makes git print non-ASCII paths verbatim instead of quoting and escaping // them (see parseDiffOutput). - out, err := exec.Command( - "git", "-c", "core.quotepath=off", "diff", "--unified=0", "--no-color", + cmd := process.Command(ctx, + "git", "-c", "core.quotepath=off", "diff", "--unified=0", "--no-color", "--src-prefix=a/", "--dst-prefix=b/", "--no-relative", "--no-ext-diff", "--no-textconv", mb, "--", "*.go", - ).Output() + ) + cmd.Dir = root + out, err := cmd.Output() if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return nil, fmt.Errorf("git diff failed: %s", strings.TrimSpace(string(exitErr.Stderr))) @@ -51,28 +63,30 @@ func ParseGitDiff(baseRef string) (ChangedLines, error) { } cl := parseDiffOutput(out, root) - if err := addUntrackedFiles(cl, root); err != nil { + if err := addUntrackedFiles(ctx, cl, root); err != nil { return nil, err } - return cl, nil + return cl, ctx.Err() } // addUntrackedFiles marks every untracked (but not ignored) Go file as a // whole-file change in cl, using the nil line set convention described on // ChangedLines. git diff does not report untracked files, so without this // step brand-new files would silently escape diff mode. -func addUntrackedFiles(cl ChangedLines, root string) error { - out, err := exec.Command( +func addUntrackedFiles(ctx context.Context, cl ChangedLines, root string) error { + cmd := process.Command(ctx, "git", "-c", "core.quotepath=off", "ls-files", - "--others", "--exclude-standard", "--full-name", "--", "*.go", - ).Output() + "-z", "--others", "--exclude-standard", "--full-name", "--", "*.go", + ) + cmd.Dir = root + out, err := cmd.Output() if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return fmt.Errorf("git ls-files failed: %s", strings.TrimSpace(string(exitErr.Stderr))) } return fmt.Errorf("git ls-files: %w", err) } - for line := range strings.SplitSeq(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\x00") { if line == "" { continue } @@ -89,8 +103,9 @@ func addUntrackedFiles(cl ChangedLines, root string) error { // report the logical, symlink-preserving one. The two views differ e.g. under // the macOS temp dir (/var -> /private/var), and ChangedLines keys must match // the Go toolchain's view for FilterPoints to work. -func repoRoot() (string, error) { - out, err := exec.Command("git", "rev-parse", "--show-cdup").Output() +func repoRoot() (string, error) { return repoRootContext(context.Background()) } +func repoRootContext(ctx context.Context) (string, error) { + out, err := process.Command(ctx, "git", "rev-parse", "--show-cdup").Output() if err != nil { return "", fmt.Errorf("not a git repository") } @@ -108,9 +123,8 @@ func repoRoot() (string, error) { // are never attributed to a stale path. // // The output must come from git run with -c core.quotepath=off so that -// non-ASCII paths appear verbatim. Known limitation: paths containing double -// quotes or control characters are quoted by git even with quotepath=off and -// are not recognized here. +// non-ASCII paths appear verbatim. Quoted paths with escaped quotes or +// control characters are decoded before resolving their absolute paths. func parseDiffOutput(output []byte, root string) ChangedLines { cl := make(ChangedLines) var currentFile string @@ -120,6 +134,11 @@ func parseDiffOutput(output []byte, root string) ChangedLines { // git appends a TAB after header paths that contain spaces // (GNU diff convention). p = strings.TrimSuffix(p, "\t") + if strings.HasPrefix(p, "\"") { + if decoded, err := strconv.Unquote(p); err == nil { + p = decoded + } + } if p == "/dev/null" { currentFile = "" } else { diff --git a/diff/git_test.go b/diff/git_test.go index 099eaad..b1a1bdd 100644 --- a/diff/git_test.go +++ b/diff/git_test.go @@ -1,6 +1,7 @@ package diff import ( + "context" "os" "os/exec" "path/filepath" @@ -490,3 +491,34 @@ func TestParseGitDiff_SymlinkedWorkingDirectory(t *testing.T) { checkChangedLines(t, link, cl, map[string][]int{"pkg/lib.go": {5}}, nil, nil) } + +func TestParseGitDiff_ConfigAndQuotedPaths(t *testing.T) { + for _, setting := range []string{"diff.mnemonicPrefix", "diff.noprefix", "diff.relative"} { + t.Run(setting, func(t *testing.T) { + const name = "other/a\"b.go" + dir := setupRepo(t, map[string]string{name: "package p\nvar A = 1\n", "sub/keep.go": "package sub\n"}) + runGit(t, dir, "config", setting, "true") + writeRepoFile(t, dir, name, "package p\nvar A = 2\n") + writeRepoFile(t, dir, "other/new\"file.go", "package p\n") + chdirRepo(t, filepath.Join(dir, "sub")) + cl, err := ParseGitDiff("base") + if err != nil { + t.Fatal(err) + } + if !cl[filepath.Join(dir, name)][2] { + t.Fatalf("missing tracked change: %#v", cl) + } + if lines, ok := cl[filepath.Join(dir, "other/new\"file.go")]; !ok || lines != nil { + t.Fatalf("missing untracked change: %#v", cl) + } + }) + } +} + +func TestParseGitDiff_Canceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := ParseGitDiffContext(ctx, "base"); err != context.Canceled { + t.Fatalf("got %v", err) + } +} diff --git a/engine/engine.go b/engine/engine.go index ddf61f7..be6e674 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -2,18 +2,23 @@ package engine import ( "bytes" + "context" "encoding/json" "errors" "fmt" "go/ast" + "go/importer" "go/parser" "go/token" + "go/types" "go/version" + "io" "os" "os/exec" "path/filepath" "strings" + "github.com/fchimpan/mutest/internal/process" "github.com/fchimpan/mutest/mutator" ) @@ -24,9 +29,13 @@ type Overlay struct { // goPackage represents a subset of `go list -json` output. type goPackage struct { - Dir string `json:"Dir"` - ImportPath string `json:"ImportPath"` - GoFiles []string `json:"GoFiles"` + Dir string `json:"Dir"` + ImportPath string `json:"ImportPath"` + GoFiles []string `json:"GoFiles"` + CgoFiles []string + ImportMap map[string]string + Imports []string + Export string Module *goModule `json:"Module"` } @@ -38,7 +47,7 @@ type goModule struct { } // minTargetGoVersion is the lowest `go` directive mutest supports in target -// modules: the generated cmp.Ordered helpers require generics (Go 1.18+), +// modules: the generated ordered-comparison helpers require generics (Go 1.18+), // and 1.20 is mutest's published floor. const minTargetGoVersion = "go1.20" @@ -67,6 +76,7 @@ type Engine struct { patterns []string // package patterns (e.g. "./...", "./pkg/calc") sourceCache map[string][]byte // file path → source bytes importPaths map[string]string // file path → import path + packages []goPackage } // New creates an Engine for the given package patterns with the given mutators. @@ -83,58 +93,155 @@ func New(patterns []string, mutators ...mutator.Mutator) *Engine { // DiscoverAll resolves the package patterns via `go list`, parses all // non-test .go files, and returns all mutation points found. func (e *Engine) DiscoverAll() ([]mutator.MutationPoint, error) { - files, err := e.resolveFiles() + return e.DiscoverAllContext(context.Background()) +} + +func (e *Engine) DiscoverAllContext(ctx context.Context) ([]mutator.MutationPoint, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if _, err := e.resolveFilesContext(ctx); err != nil { + return nil, err + } + // Parse before loading exports: malformed files must never disappear silently. + fset := token.NewFileSet() + parsed := make(map[string][]*ast.File) + paths := make(map[*ast.File]string) + for _, pkg := range e.packages { + for _, name := range append(append([]string{}, pkg.GoFiles...), pkg.CgoFiles...) { + if err := ctx.Err(); err != nil { + return nil, err + } + path := filepath.Join(pkg.Dir, name) + src, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + file, err := parser.ParseFile(fset, path, src, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + parsed[pkg.ImportPath] = append(parsed[pkg.ImportPath], file) + paths[file] = path + e.sourceCache[path] = src + } + } + imp, err := e.typeImporter(ctx, fset) if err != nil { return nil, err } - + archOut, err := process.Command(ctx, "go", "env", "GOARCH").Output() + if err != nil { + return nil, fmt.Errorf("go env GOARCH: %w", err) + } + sizes := types.SizesFor("gc", strings.TrimSpace(string(archOut))) + if sizes == nil { + return nil, fmt.Errorf("unsupported target architecture %q", strings.TrimSpace(string(archOut))) + } var points []mutator.MutationPoint - for _, path := range files { - src, err := os.ReadFile(path) - if err != nil { - continue // skip unreadable files + for _, pkg := range e.packages { + if err := ctx.Err(); err != nil { + return nil, err + } + info := &types.Info{Types: make(map[ast.Expr]types.TypeAndValue), Uses: make(map[*ast.Ident]types.Object), Defs: make(map[*ast.Ident]types.Object)} + cfg := types.Config{Importer: mappedImporter{imp, pkg.ImportMap}, FakeImportC: true, Sizes: sizes, Error: func(error) {}} + _, typeErr := cfg.Check(pkg.ImportPath, fset, parsed[pkg.ImportPath], info) + // FakeImportC cannot fully type-check C declarations. Keep unknown sites + // rather than using incomplete type information to exclude them. + if typeErr != nil && len(pkg.CgoFiles) == 0 { + return nil, fmt.Errorf("type check %s: %w", pkg.ImportPath, typeErr) } + for _, file := range parsed[pkg.ImportPath] { + path := paths[file] + si := buildSkipInfo(fset, file, e.sourceCache[path]) + for _, m := range e.mutators { + var pts []mutator.MutationPoint + if typed, ok := m.(interface { + DiscoverTyped(*token.FileSet, *ast.File, string, string, *types.Info) []mutator.MutationPoint + }); ok { + pts = typed.DiscoverTyped(fset, file, path, file.Name.Name, info) + } else { + pts = m.Discover(fset, file, path, file.Name.Name) + } + for i := range pts { + pts[i].ImportPath = pkg.ImportPath + pts[i].MutatorName = m.Name() + if !si.shouldSkip(pts[i].Line) { + points = append(points, pts[i]) + } + } + } + } + } + return points, ctx.Err() +} - fset := token.NewFileSet() - file, err := parser.ParseFile(fset, path, src, parser.ParseComments) +// Load dependency export data using the active Go toolchain. This supports +// module replacements and build tags without adding a package-loading dependency. +func (e *Engine) typeImporter(ctx context.Context, fset *token.FileSet) (types.Importer, error) { + hasImports := false + for _, pkg := range e.packages { + for _, path := range pkg.Imports { + if path != "C" && path != "unsafe" { + hasImports = true + } + } + } + exports := make(map[string]string) + if hasImports { + // Resolve from the original patterns. Synthesized import paths from + // GOPATH relative imports cannot be passed back as package arguments. + args := append([]string{"list", "-deps", "-export", "-json"}, e.patterns...) + out, err := process.Command(ctx, "go", args...).Output() if err != nil { - continue // skip unparseable files + return nil, goListError(err) } - - e.sourceCache[path] = src - pkg := file.Name.Name - importPath := e.importPaths[path] - si := buildSkipInfo(fset, file, src) - - for _, m := range e.mutators { - pts := m.Discover(fset, file, path, pkg) - for i := range pts { - pts[i].ImportPath = importPath - pts[i].MutatorName = m.Name() - if !si.shouldSkip(pts[i].Line) { - points = append(points, pts[i]) - } + dec := json.NewDecoder(bytes.NewReader(out)) + for dec.More() { + var p goPackage + if err := dec.Decode(&p); err != nil { + return nil, err } + exports[p.ImportPath] = p.Export + } + } + return importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { + if exports[path] == "" { + return nil, fmt.Errorf("no export data for %s", path) } + return os.Open(exports[path]) + }), nil +} + +// GOPATH vendoring can map a source import to a different canonical path. +type mappedImporter struct { + base types.Importer + paths map[string]string +} + +func (m mappedImporter) Import(path string) (*types.Package, error) { + if mapped := m.paths[path]; mapped != "" { + path = mapped } + return m.base.Import(path) +} - return points, nil +func goListError(err error) error { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("go list: %s", bytes.TrimSpace(exitErr.Stderr)) + } + return err } -// resolveFiles uses `go list -json` to resolve package patterns to -// absolute file paths of non-test .go files. -func (e *Engine) resolveFiles() ([]string, error) { +func (e *Engine) resolveFiles() ([]string, error) { return e.resolveFilesContext(context.Background()) } +func (e *Engine) resolveFilesContext(ctx context.Context) ([]string, error) { args := append([]string{"list", "-json"}, e.patterns...) - cmd := exec.Command("go", args...) - out, err := cmd.Output() + out, err := process.Command(ctx, "go", args...).Output() if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return nil, fmt.Errorf("go list: %s", bytes.TrimSpace(exitErr.Stderr)) - } - return nil, err + return nil, goListError(err) } - + e.packages = nil var files []string dec := json.NewDecoder(bytes.NewReader(out)) for dec.More() { @@ -145,13 +252,14 @@ func (e *Engine) resolveFiles() ([]string, error) { if err := checkGoVersion(pkg.Module); err != nil { return nil, err } - for _, f := range pkg.GoFiles { - absPath := filepath.Join(pkg.Dir, f) - files = append(files, absPath) - e.importPaths[absPath] = pkg.ImportPath + e.packages = append(e.packages, pkg) + for _, f := range append(append([]string{}, pkg.GoFiles...), pkg.CgoFiles...) { + path := filepath.Join(pkg.Dir, f) + files = append(files, path) + e.importPaths[path] = pkg.ImportPath } } - return files, nil + return files, ctx.Err() } // --- //mutest:skip directive support --- @@ -189,8 +297,8 @@ func buildSkipInfo(fset *token.FileSet, file *ast.File, src []byte) *skipInfo { } for _, c := range fn.Doc.List { if strings.Contains(c.Text, "mutest:skip") { - start := fset.Position(fn.Pos()).Line - end := fset.Position(fn.End()).Line + start := fset.PositionFor(fn.Pos(), false).Line + end := fset.PositionFor(fn.End(), false).Line si.ranges = append(si.ranges, lineRange{start, end}) break } @@ -206,7 +314,7 @@ func buildSkipInfo(fset *token.FileSet, file *ast.File, src []byte) *skipInfo { if !strings.Contains(c.Text, "mutest:skip") { continue } - line := fset.Position(c.Pos()).Line + line := fset.PositionFor(c.Pos(), false).Line si.lines[line] = true target := line @@ -243,7 +351,7 @@ func isStandaloneComment(fset *token.FileSet, pos token.Pos, src []byte) bool { if tf == nil { return false } - lineStart := tf.Offset(tf.LineStart(tf.Line(pos))) + lineStart := tf.Offset(tf.LineStart(tf.PositionFor(pos, false).Line)) commentStart := tf.Offset(pos) return len(bytes.TrimSpace(src[lineStart:commentStart])) == 0 } @@ -272,8 +380,8 @@ func buildConstRanges(fset *token.FileSet, file *ast.File) []lineRange { if !ok || decl.Tok != token.CONST { return true } - start := fset.Position(decl.Pos()).Line - end := fset.Position(decl.End()).Line + start := fset.PositionFor(decl.Pos(), false).Line + end := fset.PositionFor(decl.End(), false).Line ranges = append(ranges, lineRange{start, end}) return true }) @@ -305,8 +413,8 @@ func buildBlockRanges(fset *token.FileSet, file *ast.File) map[int]lineRange { default: return true } - line := fset.Position(start).Line - ranges[line] = lineRange{line, fset.Position(end).Line} + line := fset.PositionFor(start, false).Line + ranges[line] = lineRange{line, fset.PositionFor(end, false).Line} return true }) return ranges diff --git a/engine/instrument.go b/engine/instrument.go index 8f16368..13740de 100644 --- a/engine/instrument.go +++ b/engine/instrument.go @@ -4,17 +4,20 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "go/ast" "go/parser" "go/token" "os" - "os/exec" "path/filepath" + "runtime" + "slices" "sort" "strings" "sync" + "github.com/fchimpan/mutest/internal/process" "github.com/fchimpan/mutest/mutator" ) @@ -56,7 +59,13 @@ func (e *Engine) InstrumentAll(points []mutator.MutationPoint) (map[string]*Inst result := make(map[string]*InstrumentedPackage, len(byPkg)) - for importPath, indices := range byPkg { + importPaths := make([]string, 0, len(byPkg)) + for importPath := range byPkg { + importPaths = append(importPaths, importPath) + } + sort.Strings(importPaths) + for _, importPath := range importPaths { + indices := byPkg[importPath] // Assign MutestIDs (1-based) directly into the original slice. pkgPoints := make([]mutator.MutationPoint, len(indices)) for rank, idx := range indices { @@ -66,6 +75,7 @@ func (e *Engine) InstrumentAll(points []mutator.MutationPoint) (map[string]*Inst pkg, err := e.instrumentPackage(importPath, pkgPoints) if err != nil { + CleanupInstrumented(result) return nil, fmt.Errorf("instrument %s: %w", importPath, err) } result[importPath] = pkg @@ -257,10 +267,13 @@ func instrumentFile(src []byte, filePath string, points []mutator.MutationPoint) // switch. A comparison also yields an untyped bool, so // defined-bool-type contexts keep compiling. callExpr = fmt.Sprintf("(%s %s %s) != _mutest_on(%d)", lhs, pt.Original.String(), rhs, pt.MutestID) + } else if pt.Constant { + // Keep arbitrary-precision constant operands in their original context. + callExpr = fmt.Sprintf("((%s %s %s) != (_mutest_on(%d) && (%s == %s)))", lhs, pt.Original.String(), rhs, pt.MutestID, lhs, rhs) } else { // Ordered mutations are not negations (`>` vs `>=`), so they go // through a generic helper instead. - callExpr = fmt.Sprintf("_mutest_cmp_%d(%s, %s)", pt.MutestID, lhs, rhs) + callExpr = fmt.Sprintf("(_mutest_cmp_%d(%s, %s) != false)", pt.MutestID, lhs, rhs) helpers = append(helpers, helperSpec{ID: pt.MutestID, Original: pt.Original, Mutated: pt.Mutated}) } @@ -293,8 +306,7 @@ func instrumentFile(src []byte, filePath string, points []mutator.MutationPoint) var buf bytes.Buffer buf.Grow(len(src) * 2) pos := 0 - for i := len(rootRepls) - 1; i >= 0; i-- { - r := rootRepls[i] + for _, r := range slices.Backward(rootRepls) { buf.Write(src[pos:r.start]) buf.WriteString(r.text) pos = r.end @@ -360,9 +372,6 @@ func generateRuntime(pkg string, helpers []helperSpec) []byte { b.WriteString("package " + pkg + "\n\n") b.WriteString("import (\n") - if len(helpers) > 0 { - b.WriteString("\t\"cmp\"\n") - } b.WriteString("\t\"os\"\n") b.WriteString("\t\"strconv\"\n") b.WriteString("\t\"sync\"\n") @@ -387,9 +396,12 @@ func generateRuntime(pkg string, helpers []helperSpec) []byte { b.WriteString("\treturn _mutest_active == id\n") b.WriteString("}\n") + if len(helpers) > 0 { + b.WriteString("type _mutest_ordered interface { ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr | ~float32 | ~float64 | ~string }\n") + } for _, h := range helpers { b.WriteString("\n") - fmt.Fprintf(&b, "func _mutest_cmp_%d[T cmp.Ordered](a, b T) bool {\n", h.ID) + fmt.Fprintf(&b, "func _mutest_cmp_%d[T _mutest_ordered](a, b T) bool {\n", h.ID) b.WriteString("\t_mutest_init()\n") fmt.Fprintf(&b, "\tif _mutest_active == %d {\n", h.ID) fmt.Fprintf(&b, "\t\treturn a %s b\n", h.Mutated.String()) @@ -405,8 +417,8 @@ func generateRuntime(pkg string, helpers []helperSpec) []byte { func (e *Engine) BuildTestBinary(ctx context.Context, pkg *InstrumentedPackage) error { binPath := filepath.Join(pkg.TempDir, "pkg.test") args := []string{"test", "-c", "-overlay=" + pkg.OverlayPath, "-vet=off", "-p=1", "-ldflags=-s -w", "-o", binPath, pkg.ImportPath} - cmd := exec.CommandContext(ctx, "go", args...) - out, err := cmd.CombinedOutput() + cmd := process.Command(ctx, "go", args...) + out, err := process.CombinedOutput(cmd, 64<<10) if err != nil { return fmt.Errorf("build %s: %w\n%s", pkg.ImportPath, err, out) } @@ -414,6 +426,9 @@ func (e *Engine) BuildTestBinary(ctx context.Context, pkg *InstrumentedPackage) // test files (it prints "? pkg [no test files]"). Detect this via stat — // never by parsing output — and mark the package so its mutants survive. if _, statErr := os.Stat(binPath); statErr != nil { + if !errors.Is(statErr, os.ErrNotExist) { + return statErr + } pkg.NoTests = true return nil } @@ -422,26 +437,47 @@ func (e *Engine) BuildTestBinary(ctx context.Context, pkg *InstrumentedPackage) } // BuildTestBinaries builds test binaries for all packages in parallel. -func (e *Engine) BuildTestBinaries(ctx context.Context, pkgs map[string]*InstrumentedPackage) error { +func (e *Engine) BuildTestBinaries(ctx context.Context, pkgs map[string]*InstrumentedPackage, limits ...int) error { + workers := runtime.NumCPU() + if len(limits) > 0 { + workers = limits[0] + } + if workers < 1 { + return fmt.Errorf("build workers must be positive") + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + sem := make(chan struct{}, workers) var mu sync.Mutex var firstErr error var wg sync.WaitGroup +launch: for _, pkg := range pkgs { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break launch + } wg.Add(1) go func(p *InstrumentedPackage) { defer wg.Done() + defer func() { <-sem }() if err := e.BuildTestBinary(ctx, p); err != nil { mu.Lock() if firstErr == nil { firstErr = err + cancel() } mu.Unlock() } }(pkg) } wg.Wait() - return firstErr + if firstErr != nil { + return firstErr + } + return ctx.Err() } // CleanupInstrumented removes temp directories for all instrumented packages. diff --git a/engine/instrument_test.go b/engine/instrument_test.go index f23041d..5f8a972 100644 --- a/engine/instrument_test.go +++ b/engine/instrument_test.go @@ -61,7 +61,7 @@ func Foo(a, b int, flag bool) bool { // Both mutations should be instrumented, the inner > call nested inside // the outer == flip. - if !strings.Contains(result, "((_mutest_cmp_1(a, b)) == flag) != _mutest_on(2)") { + if !strings.Contains(result, "(((_mutest_cmp_1(a, b) != false)) == flag) != _mutest_on(2)") { t.Errorf("expected inner > call nested in outer == flip, got:\n%s", result) } @@ -179,7 +179,7 @@ func Baz(a, b int, flag, expected bool) bool { mustParseInstrumented(t, out) // All three mutations should be present, nested innermost to outermost. - want := "((((_mutest_cmp_1(a, b)) == flag) != _mutest_on(2)) != expected) != _mutest_on(3)" + want := "(((((_mutest_cmp_1(a, b) != false)) == flag) != _mutest_on(2)) != expected) != _mutest_on(3)" if !strings.Contains(result, want) { t.Errorf("expected %s, got:\n%s", want, result) } @@ -440,3 +440,37 @@ func Simple(a, b int) bool { t.Errorf("expected 1 helper, got %d", len(helpers)) } } + +func TestInstrumentAll_CleansPartialFailure(t *testing.T) { + root := t.TempDir() + temp := t.TempDir() + t.Setenv("TMPDIR", temp) + t.Setenv("TMP", temp) + t.Setenv("TEMP", temp) + e := New(nil) + var points []mutator.MutationPoint + for _, name := range []string{"a", "z"} { + dir := filepath.Join(root, name) + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "lib.go") + e.sourceCache[path] = []byte("package p\nfunc F(n int) bool {return n>0}\n") + points = append(points, mutator.MutationPoint{File: path, Package: "p", ImportPath: name, NodeID: 0, Original: token.GTR, Mutated: token.GEQ}) + } + if err := os.WriteFile(filepath.Join(root, "z", "mutest_runtime.go"), []byte("package p"), 0644); err != nil { + t.Fatal(err) + } + pkgs, err := e.InstrumentAll(points) + if err == nil { + CleanupInstrumented(pkgs) + t.Fatal("expected collision") + } + entries, err := os.ReadDir(temp) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("leaked temporary directories: %v", entries) + } +} diff --git a/engine/integration_test.go b/engine/integration_test.go new file mode 100644 index 0000000..269c450 --- /dev/null +++ b/engine/integration_test.go @@ -0,0 +1,39 @@ +package engine_test + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/fchimpan/mutest/engine" + "github.com/fchimpan/mutest/mutator" +) + +func TestEqualityInstrumentationDoesNotReserveUnusedOrderedName(t *testing.T) { + dir := t.TempDir() + for name, src := range map[string]string{ + "go.mod": "module p\ngo 1.24.0\n", + "lib.go": "package p\ntype _mutest_ordered int\nfunc F(n int)bool{return n==0}\n", + "lib_test.go": "package p\nimport \"testing\"\nfunc TestF(t *testing.T){if !F(0){t.Fail()}}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + } + chdir(t, dir) + + e := engine.New([]string{"."}, &mutator.EqualityMutator{}) + points, err := e.DiscoverAll() + if err != nil { + t.Fatal(err) + } + pkgs, err := e.InstrumentAll(points) + if err != nil { + t.Fatal(err) + } + defer engine.CleanupInstrumented(pkgs) + if err := e.BuildTestBinaries(context.Background(), pkgs); err != nil { + t.Fatalf("unused helper name collided with user declaration: %v", err) + } +} diff --git a/engine/process_unix_test.go b/engine/process_unix_test.go new file mode 100644 index 0000000..b7e954e --- /dev/null +++ b/engine/process_unix_test.go @@ -0,0 +1,73 @@ +//go:build unix + +package engine_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/fchimpan/mutest/engine" +) + +func fakeGo(t *testing.T, script string) { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "go"), []byte("#!/bin/sh\n"+script), 0755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} + +func TestDiscoveryCancelsRunningGoList(t *testing.T) { + fakeGo(t, "sleep 30\n") + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + _, err := engine.New([]string{"."}).DiscoverAllContext(ctx) + if err == nil || ctx.Err() == nil || time.Since(start) > 2*time.Second { + t.Fatalf("cancellation: %v after %v", err, time.Since(start)) + } +} + +func TestBuildTestBinariesBoundsConcurrency(t *testing.T) { + t.Setenv("MUTEST_BUILD_LOCK", filepath.Join(t.TempDir(), "build.lock")) + fakeGo(t, `mkdir "$MUTEST_BUILD_LOCK" || exit 1 +trap 'rmdir "$MUTEST_BUILD_LOCK"' EXIT +sleep 0.1 +while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then shift; touch "$1"; break; fi + shift +done +`) + pkgs := map[string]*engine.InstrumentedPackage{} + for _, name := range []string{"a", "b", "c"} { + pkgs[name] = &engine.InstrumentedPackage{ImportPath: name, TempDir: t.TempDir()} + } + if err := engine.New(nil).BuildTestBinaries(context.Background(), pkgs, 1); err != nil { + t.Fatal(err) + } + for _, pkg := range pkgs { + if pkg.BinaryPath == "" { + t.Fatalf("package not built: %+v", pkg) + } + } +} + +func TestBuildTestBinariesPreservesBuildError(t *testing.T) { + fakeGo(t, "echo deliberate-build-failure >&2\nexit 7\n") + pkgs := map[string]*engine.InstrumentedPackage{ + "p": { + ImportPath: "p", + TempDir: t.TempDir(), + }, + } + // Exercise the existing API with no optional worker limit. + err := engine.New(nil).BuildTestBinaries(context.Background(), pkgs) + if err == nil || !strings.Contains(err.Error(), "deliberate-build-failure") { + t.Fatalf("original build failure lost: %v", err) + } +} diff --git a/engine/regression_test.go b/engine/regression_test.go new file mode 100644 index 0000000..14d5312 --- /dev/null +++ b/engine/regression_test.go @@ -0,0 +1,147 @@ +package engine_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/fchimpan/mutest/engine" + "github.com/fchimpan/mutest/mutator" +) + +func TestDiscoverDependencyConstants(t *testing.T) { + for _, mode := range []string{"replacement", "gopath vendor"} { + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + t.Setenv("GOWORK", "off") + var dir string + files := map[string]string{} + if mode == "replacement" { + t.Setenv("GO111MODULE", "on") + dir = filepath.Join(root, "app") + files["app/go.mod"] = "module example.org/app\ngo 1.24.0\nrequire example.org/dep v0.0.0\nreplace example.org/dep => ../dep\n" + files["dep/go.mod"] = "module example.org/dep\ngo 1.24.0\n" + files["dep/dep.go"] = "package dep\nconst Huge = 1 << 100\n" + } else { + t.Setenv("GO111MODULE", "off") + t.Setenv("GOPATH", root) + dir = filepath.Join(root, "src/example.org/app") + files["src/example.org/app/vendor/example.org/dep/dep.go"] = "package dep\nconst Huge = 1 << 100\n" + } + for path, src := range files { + path = filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(src), 0644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(dir, "app.go"), []byte("package app\nimport \"example.org/dep\"\nfunc F() bool {return dep.Huge > 0}\n"), 0644); err != nil { + t.Fatal(err) + } + chdir(t, dir) + e := engine.New([]string{"."}, &mutator.ComparisonMutator{}) + points, err := e.DiscoverAll() + if err != nil { + t.Fatal(err) + } + if len(points) != 1 || !points[0].Constant { + t.Fatalf("constant metadata lost: %+v", points) + } + }) + } +} + +func TestDiscoverRelativeImports(t *testing.T) { + dir := t.TempDir() + t.Setenv("GO111MODULE", "off") + t.Setenv("GOWORK", "off") + if err := os.Mkdir(filepath.Join(dir, "dep"), 0755); err != nil { + t.Fatal(err) + } + for name, src := range map[string]string{ + "lib.go": `package p +import "./dep" +func F() bool {return dep.Huge > 0} +`, + "dep/dep.go": "package dep\nconst Huge=1<<100\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0644); err != nil { + t.Fatal(err) + } + } + chdir(t, dir) + points, err := engine.New([]string{"./..."}, &mutator.ComparisonMutator{}).DiscoverAll() + if err != nil { + t.Fatal(err) + } + if len(points) != 1 || !points[0].Constant { + t.Fatalf("wrong discovery: %+v", points) + } +} + +func TestDiscoverUsesGoEnvArchitecture(t *testing.T) { + dir := t.TempDir() + for name, src := range map[string]string{ + "go.mod": "module p\ngo 1.24.0\n", + "goenv": "GOARCH=386\nGOOS=linux\n", + "lib.go": `package p +import "unsafe" +func F(x int) int {if x missing}\n", + } + for name, src := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0644); err != nil { + t.Fatal(err) + } + } + chdir(t, dir) + _, err := engine.New([]string{"."}, &mutator.ComparisonMutator{}).DiscoverAll() + if err == nil || !strings.Contains(err.Error(), "missing") { + t.Fatalf("invalid package accepted: %v", err) + } +} + +func chdir(t *testing.T, dir string) { + t.Helper() + original, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chdir(original); err != nil { + t.Errorf("restore working directory: %v", err) + } + }) +} diff --git a/internal/process/process.go b/internal/process/process.go new file mode 100644 index 0000000..1729c2c --- /dev/null +++ b/internal/process/process.go @@ -0,0 +1,55 @@ +// Package process provides cancellation and bounded diagnostic output for subprocesses. +package process + +import ( + "context" + "os/exec" + "sync" + "time" +) + +func Command(ctx context.Context, name string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, name, args...) + cmd.WaitDelay = time.Second + configure(cmd) + return cmd +} + +// CombinedOutput retains only the last limit bytes, even if a test logs forever. +func CombinedOutput(cmd *exec.Cmd, limit int) ([]byte, error) { + b := &tailBuffer{limit: limit} + cmd.Stdout, cmd.Stderr = b, b + err := cmd.Run() + // Reap any descendants that outlived the direct child. + if cmd.Process != nil { + _ = cmd.Cancel() + } + if b.truncated { + return append([]byte("...(truncated)...\n"), b.data...), err + } + return b.data, err +} + +type tailBuffer struct { + mu sync.Mutex + data []byte + limit int + truncated bool +} + +func (b *tailBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + n := len(p) + if len(b.data)+n > b.limit { + b.truncated = true + // At n == limit both branches discard all old bytes and retain p. + if n >= b.limit { //mutest:skip + b.data = append(b.data[:0], p[n-b.limit:]...) + return n, nil + } + b.data = append(b.data[:0], b.data[len(b.data)+n-b.limit:]...) + } + b.data = append(b.data, p...) + return n, nil +} diff --git a/internal/process/process_other.go b/internal/process/process_other.go new file mode 100644 index 0000000..d81fbfe --- /dev/null +++ b/internal/process/process_other.go @@ -0,0 +1,7 @@ +//go:build !unix && !windows + +package process + +import "os/exec" + +func configure(cmd *exec.Cmd) {} diff --git a/internal/process/process_test.go b/internal/process/process_test.go new file mode 100644 index 0000000..6604929 --- /dev/null +++ b/internal/process/process_test.go @@ -0,0 +1,57 @@ +package process + +import ( + "bytes" + "testing" +) + +func TestTailBufferBoundsMemory(t *testing.T) { + b := &tailBuffer{limit: 16} + chunks := [][]byte{ + bytes.Repeat([]byte("x"), 10000), + []byte("abcdefghijkl"), + []byte("mnop"), + } + for _, p := range chunks { + if n, err := b.Write(p); err != nil || n != len(p) { + t.Fatal(n, err) + } + } + if string(b.data) != "abcdefghijklmnop" || !b.truncated { + t.Fatalf("%q truncated=%v", b.data, b.truncated) + } +} + +func TestTailBufferExactCapacityIsNotTruncated(t *testing.T) { + tests := []struct { + name string + chunks [][]byte + }{ + { + name: "single write", + chunks: [][]byte{ + []byte("12345678"), + }, + }, + { + name: "multiple writes", + chunks: [][]byte{ + []byte("1234"), + []byte("5678"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &tailBuffer{limit: 8} + for _, p := range tt.chunks { + if _, err := b.Write(p); err != nil { + t.Fatal(err) + } + } + if b.truncated || string(b.data) != "12345678" { + t.Fatalf("exact capacity: %q truncated=%v", b.data, b.truncated) + } + }) + } +} diff --git a/internal/process/process_unix.go b/internal/process/process_unix.go new file mode 100644 index 0000000..36c4ee3 --- /dev/null +++ b/internal/process/process_unix.go @@ -0,0 +1,21 @@ +//go:build unix + +package process + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +func configure(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err + } +} diff --git a/internal/process/process_unix_test.go b/internal/process/process_unix_test.go new file mode 100644 index 0000000..3a0fb43 --- /dev/null +++ b/internal/process/process_unix_test.go @@ -0,0 +1,39 @@ +//go:build unix + +package process + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" +) + +func TestCancellationStopsDescendants(t *testing.T) { + marker := filepath.Join(t.TempDir(), "child-survived") + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + cmd := Command(ctx, "sh", "-c", `(sleep 0.6; touch "$1") & wait`, "sh", marker) + start := time.Now() + _, err := CombinedOutput(cmd, 1024) + if err == nil || time.Since(start) > 2*time.Second { + t.Fatalf("termination: %v elapsed=%s", err, time.Since(start)) + } + time.Sleep(700 * time.Millisecond) + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatal("descendant survived cancellation") + } +} + +func TestNormalExitStopsRemainingDescendants(t *testing.T) { + marker := filepath.Join(t.TempDir(), "orphan") + cmd := Command(context.Background(), "sh", "-c", `(sleep 0.5; touch "$1") >/dev/null 2>&1 &`, "sh", marker) + if _, err := CombinedOutput(cmd, 1024); err != nil { + t.Fatal(err) + } + time.Sleep(800 * time.Millisecond) + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatal("normal parent exit left a descendant running") + } +} diff --git a/internal/process/process_windows.go b/internal/process/process_windows.go new file mode 100644 index 0000000..79bac77 --- /dev/null +++ b/internal/process/process_windows.go @@ -0,0 +1,20 @@ +package process + +import ( + "context" + "os/exec" + "strconv" + "time" +) + +func configure(cmd *exec.Cmd) { + cmd.Cancel = func() error { + // taskkill /T terminates the process tree, including tests' subprocesses. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exec.CommandContext(ctx, "taskkill", "/F", "/T", "/PID", strconv.Itoa(cmd.Process.Pid)).Run(); err != nil { + return cmd.Process.Kill() + } + return nil + } +} diff --git a/main.go b/main.go index fd3a582..10b5bd7 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -23,6 +24,16 @@ func main() { if err := mutest.Run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { fmt.Fprintf(os.Stderr, "mutest: %v\n", err) - os.Exit(1) + os.Exit(exitCode(err)) } } + +func exitCode(err error) int { + if err == nil { + return 0 + } + if errors.Is(err, mutest.ErrTestsFailed) || errors.Is(err, mutest.ErrBaseline) || errors.Is(err, mutest.ErrInterrupted) { + return 1 + } + return 2 +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..24f73b9 --- /dev/null +++ b/main_test.go @@ -0,0 +1,18 @@ +package main + +import ( + "fmt" + "github.com/fchimpan/mutest/cmd/mutest" + "testing" +) + +func TestExitCodes(t *testing.T) { + for _, tt := range []struct { + err error + code int + }{{nil, 0}, {mutest.ErrTestsFailed, 1}, {mutest.ErrBaseline, 1}, {mutest.ErrInterrupted, 1}, {mutest.ErrDiscovery, 2}, {mutest.ErrBuild, 2}, {fmt.Errorf("wrapped: %w", mutest.ErrDiscovery), 2}} { + if got := exitCode(tt.err); got != tt.code { + t.Errorf("%v: %d want %d", tt.err, got, tt.code) + } + } +} diff --git a/mutator/clamp.go b/mutator/clamp.go new file mode 100644 index 0000000..2cb1558 --- /dev/null +++ b/mutator/clamp.go @@ -0,0 +1,58 @@ +package mutator + +import ( + "go/ast" + "go/constant" + "go/token" + "go/types" +) + +// Only local integer variables and equal constants are proven safe here. +// Floats (signed zero), map writes, else branches and calls are observable. +func clampComparisons(file *ast.File, info *types.Info) map[*ast.BinaryExpr]bool { + skip := make(map[*ast.BinaryExpr]bool) + if info == nil { + return skip + } + ast.Inspect(file, func(n ast.Node) bool { + stmt, ok := n.(*ast.IfStmt) + if !ok || stmt.Else != nil || len(stmt.Body.List) != 1 { + return true + } + bin, ok := ast.Unparen(stmt.Cond).(*ast.BinaryExpr) + if !ok { + return true + } + if _, ok := swapTable[bin.Op]; !ok { + return true + } + assign, ok := stmt.Body.List[0].(*ast.AssignStmt) + if !ok || assign.Tok != token.ASSIGN || len(assign.Lhs) != 1 || len(assign.Rhs) != 1 { + return true + } + dest, ok := ast.Unparen(assign.Lhs[0]).(*ast.Ident) + if !ok { + return true + } + obj, ok := info.ObjectOf(dest).(*types.Var) + if !ok || obj.Pkg() == nil || obj.Parent() == obj.Pkg().Scope() { + return true + } + typ, ok := obj.Type().Underlying().(*types.Basic) + if !ok || typ.Info()&types.IsInteger == 0 { + return true + } + for _, pair := range [][2]ast.Expr{{bin.X, bin.Y}, {bin.Y, bin.X}} { + id, ok := ast.Unparen(pair[0]).(*ast.Ident) + if !ok || info.ObjectOf(id) != obj { + continue + } + a, b := info.Types[pair[1]].Value, info.Types[assign.Rhs[0]].Value + if a != nil && b != nil && constant.Compare(a, token.EQL, b) { + skip[bin] = true + } + } + return true + }) + return skip +} diff --git a/mutator/clamp_test.go b/mutator/clamp_test.go new file mode 100644 index 0000000..4c6764e --- /dev/null +++ b/mutator/clamp_test.go @@ -0,0 +1,112 @@ +package mutator_test + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "testing" + + "github.com/fchimpan/mutest/mutator" +) + +func TestClampOnlySkipsProvenIntegerAssignments(t *testing.T) { + for _, tt := range []struct { + name string + body string + want int + }{ + { + name: "lower", + body: "func F(x int) int {if x<1 {x=1};return x}", + }, + { + name: "lower inclusive", + body: "func F(x int) int {if x<=1 {x=1};return x}", + }, + { + name: "upper", + body: "func F(x int) int {if x>1 {x=1};return x}", + }, + { + name: "upper inclusive", + body: "func F(x int) int {if x>=1 {x=1};return x}", + }, + { + name: "swapped lower", + body: "func F(x int) int {if 1>x {x=1};return x}", + }, + { + name: "swapped lower inclusive", + body: "func F(x int) int {if 1>=x {x=1};return x}", + }, + { + name: "swapped upper", + body: "func F(x int) int {if 1 0 → len(x) >= 0 (>= 0 is always true, not a useful boundary) -// len(x) >= 0 → len(x) > 0 (narrows from always-true, but not a boundary bug) -// len(x) < 0 → len(x) <= 0 (< 0 is always false) -// len(x) <= 0 → len(x) < 0 (always-false vs empty-check, not a boundary) -func isNonNegativeComparison(bin *ast.BinaryExpr) bool { - return (isNonNegativeCall(bin.X) && isIntLit(bin.Y, "0")) || - (isNonNegativeCall(bin.Y) && isIntLit(bin.X, "0")) -} - -func isNonNegativeCall(expr ast.Expr) bool { - call, ok := expr.(*ast.CallExpr) - if !ok { - return false - } - ident, ok := call.Fun.(*ast.Ident) - if !ok { - return false - } - return ident.Name == "len" || ident.Name == "cap" -} - -func isIntLit(expr ast.Expr, value string) bool { - lit, ok := expr.(*ast.BasicLit) - return ok && lit.Kind == token.INT && lit.Value == value -} - func (m *ComparisonMutator) Apply(file *ast.File, point MutationPoint) { done := false nodeID := 0 diff --git a/mutator/comparison_test.go b/mutator/comparison_test.go index 77530cf..248c13d 100644 --- a/mutator/comparison_test.go +++ b/mutator/comparison_test.go @@ -105,7 +105,7 @@ func f(a, b int) bool { return a == b || a != b } } } -func TestComparisonMutator_Discover_SkipsLenCapZero(t *testing.T) { +func TestComparisonMutator_Discover_KeepsLenCapZero(t *testing.T) { src := `package example func f(s []int) { @@ -128,10 +128,9 @@ func f(s []int) { m := &ComparisonMutator{} points := m.Discover(fset, file, "/fake/test.go", "example") - // Skipped (6): len(s)>0, len(s)>=0, len(s)<0, len(s)<=0, cap(s)>0, 01 (non-zero literal), len(s)>len(s) (no literal 0) - if len(points) != 2 { - t.Errorf("expected 2 mutation points, got %d", len(points)) + // Empty inputs distinguish strict and inclusive zero boundaries. + if len(points) != 8 { + t.Errorf("expected 8 mutation points, got %d", len(points)) for i, p := range points { t.Logf(" point[%d]: line %d col %d %s", i, p.Line, p.Column, p.Desc) } diff --git a/mutator/equality.go b/mutator/equality.go index 48af816..3f3a433 100644 --- a/mutator/equality.go +++ b/mutator/equality.go @@ -3,6 +3,9 @@ package mutator import ( "go/ast" "go/token" + "go/types" + "strconv" + "strings" ) // equalitySwapTable defines equality mutations. @@ -21,9 +24,13 @@ type EqualityMutator struct { func (m *EqualityMutator) Name() string { return "comparison-equality" } func (m *EqualityMutator) Discover(fset *token.FileSet, file *ast.File, filePath, pkg string) []MutationPoint { + return m.DiscoverTyped(fset, file, filePath, pkg, nil) +} + +func (m *EqualityMutator) DiscoverTyped(fset *token.FileSet, file *ast.File, filePath, pkg string, info *types.Info) []MutationPoint { var errSkip map[*ast.BinaryExpr]bool if m.SkipErrPropagation { - errSkip = buildErrPropagationSet(file) + errSkip = buildErrPropagationSet(file, info) } var points []MutationPoint @@ -38,7 +45,7 @@ func (m *EqualityMutator) Discover(fset *token.FileSet, file *ast.File, filePath nodeID++ return true } - pos := fset.Position(bin.OpPos) + pos := fset.PositionFor(bin.OpPos, false) points = append(points, MutationPoint{ File: filePath, Package: pkg, @@ -79,14 +86,14 @@ func (m *EqualityMutator) Apply(file *ast.File, point MutationPoint) { // buildErrPropagationSet pre-walks the AST and returns a set of *ast.BinaryExpr // pointers that represent simple error propagation patterns (e.g., if err != nil { return err }). // These are skipped by default because they generate noise in mutation testing. -func buildErrPropagationSet(file *ast.File) map[*ast.BinaryExpr]bool { +func buildErrPropagationSet(file *ast.File, info *types.Info) map[*ast.BinaryExpr]bool { skip := make(map[*ast.BinaryExpr]bool) ast.Inspect(file, func(n ast.Node) bool { ifStmt, ok := n.(*ast.IfStmt) if !ok { return true } - if !isSimpleErrPropagation(ifStmt) { + if !isSimpleErrPropagation(ifStmt, info) { return true } if bin, ok := ifStmt.Cond.(*ast.BinaryExpr); ok && isErrNilCheck(bin) { @@ -112,15 +119,85 @@ func buildErrPropagationSet(file *ast.File) map[*ast.BinaryExpr]bool { // if err != nil { rel = path } // assignment, not return // if err != nil { a(); b() } // multiple statements // if err != nil { return } else { ... } // has else -func isSimpleErrPropagation(ifStmt *ast.IfStmt) bool { +func isSimpleErrPropagation(ifStmt *ast.IfStmt, info *types.Info) bool { if ifStmt.Else != nil { return false } if len(ifStmt.Body.List) != 1 { return false } - _, isReturn := ifStmt.Body.List[0].(*ast.ReturnStmt) - return isReturn + bin, ok := ifStmt.Cond.(*ast.BinaryExpr) + if !ok || bin.Op != token.NEQ || !isErrNilCheck(bin) { + return false + } + id, _ := bin.X.(*ast.Ident) + if id.Name == "nil" { + id, _ = bin.Y.(*ast.Ident) + } + if info != nil { + typ := info.TypeOf(id) + if typ == nil || !types.Implements(typ, types.Universe.Lookup("error").Type().Underlying().(*types.Interface)) { + return false + } + } + ret, ok := ifStmt.Body.List[0].(*ast.ReturnStmt) + if !ok { + return false + } + same := func(expr ast.Expr) bool { + other, ok := ast.Unparen(expr).(*ast.Ident) + if !ok || other.Name != id.Name { + return false + } + return info == nil || info.ObjectOf(other) == info.ObjectOf(id) + } + propagates := false + for _, expr := range ret.Results { + if same(expr) { + propagates = true + continue + } + // Other return values must be constants or nil; calls and computed + // values can have meaningful behavior beyond error propagation. + if info != nil { + if tv := info.Types[expr]; tv.Value != nil || tv.IsNil() { + continue + } + } else if _, ok := expr.(*ast.BasicLit); ok || isNilIdent(expr) { + continue + } + call, ok := expr.(*ast.CallExpr) + if !ok || len(call.Args) != 2 || !same(call.Args[1]) { + return false + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "Errorf" { + return false + } + if info != nil { + obj := info.ObjectOf(sel.Sel) + if obj == nil || obj.Pkg() == nil || obj.Pkg().Path() != "fmt" { + return false + } + } else { + pkg, ok := sel.X.(*ast.Ident) + if !ok || pkg.Name != "fmt" { + return false + } + } + lit, ok := call.Args[0].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return false + } + format, err := strconv.Unquote(lit.Value) + // Recognize only a single unambiguous wrapping verb. Escaped or + // indexed verbs and extra arguments remain mutation targets. + if err != nil || strings.Count(format, "%") != 1 || !strings.Contains(format, "%w") { + return false + } + propagates = true + } + return propagates } // isErrNilCheck returns true if the binary expression compares the identifier diff --git a/mutator/mutator.go b/mutator/mutator.go index 22e70bd..de276e3 100644 --- a/mutator/mutator.go +++ b/mutator/mutator.go @@ -17,6 +17,7 @@ type MutationPoint struct { NodeID int // index to identify the BinaryExpr in AST walk order Desc string // human-readable description, e.g. "> to >=" MutatorName string // name of the mutator that discovered this point + Constant bool // comparison has a compile-time constant value MutestID int // runtime mutation ID (1-based, per-package), assigned during instrumentation } diff --git a/mutator/typed_test.go b/mutator/typed_test.go new file mode 100644 index 0000000..fadb340 --- /dev/null +++ b/mutator/typed_test.go @@ -0,0 +1,96 @@ +package mutator_test + +import ( + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "testing" + + "github.com/fchimpan/mutest/mutator" +) + +func typedFile(t *testing.T, src string) (*token.FileSet, *ast.File, *types.Info) { + t.Helper() + fs := token.NewFileSet() + f, err := parser.ParseFile(fs, "p.go", "package p\n"+src, 0) + if err != nil { + t.Fatal(err) + } + info := &types.Info{Types: map[ast.Expr]types.TypeAndValue{}, Defs: map[*ast.Ident]types.Object{}, Uses: map[*ast.Ident]types.Object{}} + cfg := types.Config{Importer: importer.Default()} + if _, err := cfg.Check("p", fs, []*ast.File{f}, info); err != nil { + t.Fatal(err) + } + return fs, f, info +} + +func TestTypedComparisonDistinguishesConstants(t *testing.T) { + fs, f, info := typedFile(t, `func F(x int) bool {return 1<<100 > 0 && x < 1}`) + points := (&mutator.ComparisonMutator{}).DiscoverTyped(fs, f, "p.go", "p", info) + if len(points) != 2 || !points[0].Constant || points[1].Constant { + t.Fatalf("constant and runtime operands confused: %+v", points) + } +} + +func TestTypedErrorPropagation(t *testing.T) { + for _, tt := range []struct { + name string + src string + want int + }{ + { + name: "direct", + src: "func F(err error)error{if err!=nil{return err};return nil}", + }, + { + name: "swapped", + src: "func F(err error)error{if nil!=err{return err};return nil}", + }, + { + name: "constant result", + src: "func F(err error)(int,error){if err!=nil{return 1+2,err};return 0,nil}", + }, + { + name: "nil result", + src: "func F(err error)(*int,error){if err!=nil{return nil,err};return nil,nil}", + }, + { + name: "wrapped", + src: `import f "fmt" +func F(err error)error{if err!=nil{return f.Errorf("wrap: %w",err)};return nil}`, + }, + { + name: "non error", + src: "func F(err *int)*int{if err!=nil{return err};return nil}", + want: 1, + }, + { + name: "different error", + src: "func F(err,other error)error{if err!=nil{return other};return nil}", + want: 1, + }, + { + name: "runtime result", + src: "func F(err error,n int)(int,error){if err!=nil{return n+1,err};return 0,nil}", + want: 1, + }, + { + name: "non fmt wrapper", + src: `type formatter struct{} +func(formatter)Errorf(s string,e error)error{return e} +var fmt formatter +func F(err error)error{if err!=nil{return fmt.Errorf("wrap: %w",err)};return nil}`, + want: 1, + }, + } { + t.Run(tt.name, func(t *testing.T) { + fs, f, info := typedFile(t, tt.src) + points := (&mutator.EqualityMutator{SkipErrPropagation: true}).DiscoverTyped(fs, f, "p.go", "p", info) + if len(points) != tt.want { + t.Fatalf("got %d points, want %d", len(points), tt.want) + } + }) + } +} diff --git a/output/output.go b/output/output.go index 8bd04f5..d367839 100644 --- a/output/output.go +++ b/output/output.go @@ -94,12 +94,7 @@ func CalcKillRate(s *runner.Summary) float64 { return 0 } -// RoundedKillRate returns CalcKillRate rounded to one decimal place — the -// same precision shown in text output (%.1f) and the JSON kill_rate field. -// Any consumer that makes a pass/fail decision from the kill rate (e.g. the -// -threshold gate) must compare against this value, not CalcKillRate's raw -// result: otherwise a run that displays "Score: 80.0%" could still fail a -// "-threshold 80" gate because the unrounded rate was 79.96. +// RoundedKillRate is for display only. Quality gates use the unrounded rate. func RoundedKillRate(s *runner.Summary) float64 { return math.Round(CalcKillRate(s)*10) / 10 } diff --git a/output/reporter.go b/output/reporter.go index 2171947..01e6605 100644 --- a/output/reporter.go +++ b/output/reporter.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "strings" + "sync" "github.com/fchimpan/mutest/config" "github.com/fchimpan/mutest/mutator" @@ -18,16 +19,16 @@ type Reporter struct { stdout io.Writer stderr io.Writer rpc *RelPathCache + errors *writeErrors } // NewReporter builds a Reporter for the given config. baseDir is the directory // against which file paths are reported (typically the current working dir). func NewReporter(cfg config.Config, stdout, stderr io.Writer, baseDir string) *Reporter { + errors := &writeErrors{} return &Reporter{ - cfg: cfg, - stdout: stdout, - stderr: stderr, - rpc: NewRelPathCache(baseDir), + cfg: cfg, stdout: &checkedWriter{w: stdout, errors: errors}, stderr: &checkedWriter{w: stderr, errors: errors}, + rpc: NewRelPathCache(baseDir), errors: errors, } } @@ -112,3 +113,35 @@ func statusOf(r runner.Result) string { return "KILLED" } } + +// Err returns the first output failure, including writes through Info. +func (r *Reporter) Err() error { + r.errors.mu.Lock() + defer r.errors.mu.Unlock() + return r.errors.err +} + +type writeErrors struct { + mu sync.Mutex + err error +} +type checkedWriter struct { + w io.Writer + errors *writeErrors +} + +func (w *checkedWriter) Write(p []byte) (int, error) { + w.errors.mu.Lock() + defer w.errors.mu.Unlock() + if w.errors.err != nil { + return 0, w.errors.err + } + n, err := w.w.Write(p) + if err == nil && n != len(p) { + err = io.ErrShortWrite + } + if err != nil { + w.errors.err = err + } + return n, err +} diff --git a/output/reporter_test.go b/output/reporter_test.go index 8a204e3..3680a5b 100644 --- a/output/reporter_test.go +++ b/output/reporter_test.go @@ -2,7 +2,9 @@ package output import ( "bytes" + "errors" "fmt" + "io" "strings" "testing" "time" @@ -16,12 +18,14 @@ func TestReporter_Info(t *testing.T) { var stdout, stderr bytes.Buffer textRep := NewReporter(config.Config{}, &stdout, &stderr, "/") - if textRep.Info() != &stdout { + fmt.Fprint(textRep.Info(), "text") + if stdout.String() != "text" || stderr.Len() != 0 { t.Error("text mode: Info() should return stdout") } jsonRep := NewReporter(config.Config{JSON: true}, &stdout, &stderr, "/") - if jsonRep.Info() != &stderr { + fmt.Fprint(jsonRep.Info(), "json") + if stderr.String() != "json" || stdout.String() != "text" { t.Error("json mode: Info() should return stderr") } } @@ -201,3 +205,57 @@ func TestStatusOf(t *testing.T) { }) } } + +type failingOutput struct { + err error + short bool + calls int +} + +func (w *failingOutput) Write(p []byte) (int, error) { + w.calls++ + if w.short { + return len(p) - 1, w.err + } + return len(p), w.err +} + +func TestReporterRetainsFirstWriteFailure(t *testing.T) { + diskErr := errors.New("disk full") + for _, tt := range []struct { + name string + short bool + writeErr error + want error + }{ + { + name: "short write", + short: true, + want: io.ErrShortWrite, + }, + { + name: "partial failure", + short: true, + writeErr: diskErr, + want: diskErr, + }, + { + name: "full length failure", + writeErr: diskErr, + want: diskErr, + }, + } { + t.Run(tt.name, func(t *testing.T) { + writer := &failingOutput{err: tt.writeErr, short: tt.short} + rep := NewReporter(config.Config{JSON: true}, writer, io.Discard, "/") + rep.NoMutationPoints() + if !errors.Is(rep.Err(), tt.want) { + t.Fatalf("lost output failure: %v", rep.Err()) + } + rep.NoMutationPoints() + if writer.calls != 1 || !errors.Is(rep.Err(), tt.want) { + t.Fatalf("failure was not retained: calls=%d err=%v", writer.calls, rep.Err()) + } + }) + } +} diff --git a/runner/runner.go b/runner/runner.go index fc505d2..4df79f5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -6,11 +6,13 @@ import ( "fmt" "os" "os/exec" + "sort" "strings" "sync" "time" "github.com/fchimpan/mutest/engine" + "github.com/fchimpan/mutest/internal/process" "github.com/fchimpan/mutest/mutator" ) @@ -50,53 +52,72 @@ type ProgressFunc func(result Result, done, total int) // RunInstrumented tests all mutants using pre-built test binaries. // Each mutation is activated via MUTEST_ID env var, avoiding per-mutation compilation. func RunInstrumented(ctx context.Context, pkgs map[string]*engine.InstrumentedPackage, cfg Config, progress ProgressFunc) *Summary { - // Flatten all mutations across packages. + // Tests within a package share their working directory and fixtures. + // Run that package serially; parallelize only independent packages. + type job struct { + pkg *engine.InstrumentedPackage + offset int + } + var keys []string + for key := range pkgs { + keys = append(keys, key) + } + sort.Strings(keys) + var jobs []job var allPoints []mutator.MutationPoint - pointToPkg := make(map[int]*engine.InstrumentedPackage) // index → pkg - for _, pkg := range pkgs { - for _, pt := range pkg.Mutations { - pointToPkg[len(allPoints)] = pkg - allPoints = append(allPoints, pt) - } + for _, key := range keys { + p := pkgs[key] + jobs = append(jobs, job{p, len(allPoints)}) + allPoints = append(allPoints, p.Mutations...) } - start := time.Now() results := make([]Result, len(allPoints)) - // Prefill every result as Canceled so that any mutant we never launch - // (e.g. after ctx cancellation breaks the loop below) is counted as - // canceled rather than as a false Survived (the zero value). - for i := range results { - results[i] = Result{Point: allPoints[i], Canceled: true} + reported := make([]bool, len(allPoints)) + for i, pt := range allPoints { + results[i] = Result{Point: pt, Canceled: true} } - sem := make(chan struct{}, cfg.Workers) + sem := make(chan struct{}, max(1, cfg.Workers)) var mu sync.Mutex done := 0 - + report := func(idx int, r Result) { + mu.Lock() + defer mu.Unlock() + reported[idx] = true + done++ + if progress != nil { + progress(r, done, len(allPoints)) + } + } var wg sync.WaitGroup - for i, point := range allPoints { - if ctx.Err() != nil { - break // stop launching new mutants once the run is canceled +launch: + for _, j := range jobs { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break launch } wg.Add(1) - sem <- struct{}{} - go func(idx int, pt mutator.MutationPoint) { + go func() { defer wg.Done() defer func() { <-sem }() - - pkg := pointToPkg[idx] - r := testMutantRuntime(ctx, pkg, pt, cfg) - results[idx] = r - - if progress != nil { - mu.Lock() - done++ - d := done - progress(r, d, len(allPoints)) - mu.Unlock() + for i, pt := range j.pkg.Mutations { + if ctx.Err() != nil { + return + } + idx := j.offset + i + r := testMutantRuntime(ctx, j.pkg, pt, cfg) + results[idx] = r + report(idx, r) } - }(i, point) + }() } wg.Wait() + // NDJSON consumers also need the identities of mutants never launched. + for i, r := range results { + if !reported[i] { + report(i, r) + } + } summary := &Summary{ Total: len(allPoints), @@ -146,17 +167,22 @@ func testArgs(cfg Config) []string { // false 100% score. Packages without test files (NoTests) are skipped. // Parallelism is bounded by cfg.Workers. func VerifyBaseline(ctx context.Context, pkgs map[string]*engine.InstrumentedPackage, cfg Config) error { - sem := make(chan struct{}, cfg.Workers) + sem := make(chan struct{}, max(1, cfg.Workers)) var mu sync.Mutex var firstErr error var wg sync.WaitGroup +launch: for _, pkg := range pkgs { if pkg.NoTests { continue } + select { + case sem <- struct{}{}: + case <-ctx.Done(): + break launch + } wg.Add(1) - sem <- struct{}{} go func(p *engine.InstrumentedPackage) { defer wg.Done() defer func() { <-sem }() @@ -164,13 +190,13 @@ func VerifyBaseline(ctx context.Context, pkgs map[string]*engine.InstrumentedPac testCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) defer cancel() - cmd := exec.CommandContext(testCtx, p.BinaryPath, testArgs(cfg)...) + cmd := process.Command(testCtx, p.BinaryPath, testArgs(cfg)...) cmd.Dir = p.Dir // Neutralize any MUTEST_ID inherited from the caller's environment: // ID 0 activates no mutation (helper IDs are 1-based), and os/exec // keeps the last duplicate key, so this append always wins. cmd.Env = append(os.Environ(), "MUTEST_ID=0") - output, err := cmd.CombinedOutput() + output, err := process.CombinedOutput(cmd, 64<<10) if err != nil { mu.Lock() if firstErr == nil { @@ -181,7 +207,10 @@ func VerifyBaseline(ctx context.Context, pkgs map[string]*engine.InstrumentedPac }(pkg) } wg.Wait() - return firstErr + if firstErr != nil { + return firstErr + } + return ctx.Err() } // trimBaselineOutput trims a test binary's combined output to a readable tail @@ -198,6 +227,9 @@ func trimBaselineOutput(out []byte) string { func testMutantRuntime(ctx context.Context, pkg *engine.InstrumentedPackage, pt mutator.MutationPoint, cfg Config) Result { // A package with no test files has no binary to run; its mutants survive // (no tests is the largest possible test gap). + if ctx.Err() != nil { + return Result{Point: pt, Canceled: true} + } if pkg.NoTests { return Result{Point: pt, Output: "package has no test files"} } @@ -207,10 +239,10 @@ func testMutantRuntime(ctx context.Context, pkg *engine.InstrumentedPackage, pt testCtx, cancel := context.WithTimeout(ctx, cfg.Timeout) defer cancel() - cmd := exec.CommandContext(testCtx, pkg.BinaryPath, testArgs(cfg)...) + cmd := process.Command(testCtx, pkg.BinaryPath, testArgs(cfg)...) cmd.Dir = pkg.Dir cmd.Env = append(os.Environ(), fmt.Sprintf("MUTEST_ID=%d", pt.MutestID)) - output, err := cmd.CombinedOutput() + output, err := process.CombinedOutput(cmd, 64<<10) // Order matters: a canceled run and a timed-out run both kill the process // (yielding an *exec.ExitError), so they must be classified before the @@ -234,7 +266,11 @@ func testMutantRuntime(ctx context.Context, pkg *engine.InstrumentedPackage, pt case err == nil: // Tests passed with the mutation active: mutant survived. case errors.As(err, &exitErr): - r.Killed = true // tests ran and failed: genuine kill + if !exitErr.ProcessState.Exited() { + r.Err = err + } else { + r.Killed = true + } // External signals are not assertions. default: r.Err = err // fork failure, missing binary, etc.: ERROR, not KILLED } diff --git a/runner/runner_test.go b/runner/runner_test.go index 07de972..98486ae 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -420,3 +420,33 @@ func TestTrimBaselineOutput(t *testing.T) { }) } } + +func TestRunInstrumented_ReportsEveryCanceledMutant(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + pkgs := map[string]*engine.InstrumentedPackage{ + "a": { + NoTests: true, + Mutations: []mutator.MutationPoint{ + {MutestID: 1}, + {MutestID: 2}, + }, + }, + "b": { + NoTests: true, + Mutations: []mutator.MutationPoint{ + {MutestID: 3}, + }, + }, + } + var ids []int + s := RunInstrumented(ctx, pkgs, Config{Workers: 1}, func(r Result, done, total int) { + if !r.Canceled || done != len(ids)+1 || total != 3 { + t.Errorf("bad canceled event: %+v %d/%d", r, done, total) + } + ids = append(ids, r.Point.MutestID) + }) + if s.Canceled != 3 || len(ids) != 3 { + t.Fatalf("missing events: %+v %v", s, ids) + } +} diff --git a/runner/signals_unix_test.go b/runner/signals_unix_test.go new file mode 100644 index 0000000..09c0e8e --- /dev/null +++ b/runner/signals_unix_test.go @@ -0,0 +1,24 @@ +//go:build unix + +package runner + +import ( + "context" + "github.com/fchimpan/mutest/engine" + "github.com/fchimpan/mutest/mutator" + "os" + "path/filepath" + "testing" + "time" +) + +func TestExternalSignalIsAnError(t *testing.T) { + bin := filepath.Join(t.TempDir(), "test.sh") + if err := os.WriteFile(bin, []byte("#!/bin/sh\nkill -KILL $$\n"), 0755); err != nil { + t.Fatal(err) + } + result := testMutantRuntime(context.Background(), &engine.InstrumentedPackage{BinaryPath: bin}, mutator.MutationPoint{}, Config{Timeout: time.Second}) + if result.Err == nil || result.Killed || result.Canceled || result.TimedOut { + t.Fatalf("external signal misclassified: %+v", result) + } +}