From 4e30b685b9b0c72b188943b2a8e8117459396a93 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 17 Jul 2026 19:36:56 -0500 Subject: [PATCH 1/4] ENG-2084: Install private wheels into the Python runtime's site-packages The `state publish --build` private-ingredient consume path installed the decrypted wheel into a flat `site-packages` at the artifact deploy-tree root, which linked to `/site-packages` -- the wrong location. The wheel must land in the Python runtime's own site-packages, whose layout varies by platform and Python version (e.g. `lib/pythonX.Y/site-packages` on Linux, `lib/site-packages` on macOS). The install previously ran during the parallel unpack phase, before the Python runtime was guaranteed on disk, so the correct path could not be discovered. Defer the placement until every artifact is unpacked, then locate the real site-packages by scanning the unpacked non-private artifacts (glob known layouts, full-walk fallback) and lay the wheel out at that same relative path so the normal deploy merges it into the runtime's site-packages. PYTHONPATH now points at the discovered path, matching the runtime's separator/inherit directives. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/runtime/setup.go | 223 +++++++++++++++++++++++++++---- pkg/runtime/sitepackages_test.go | 81 +++++++++++ 2 files changed, 278 insertions(+), 26 deletions(-) create mode 100644 pkg/runtime/sitepackages_test.go diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index ded39d5324..7e84be97bc 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "os" "path/filepath" + "sort" "strings" "sync" @@ -115,6 +116,13 @@ type setup struct { // was available to decrypt them. skipMutex sync.Mutex skipped map[strfmt.UUID]struct{} + + // privateWheels records the depot directories of decrypted private wheels + // whose installation into site-packages is deferred until every artifact is + // unpacked, so the Python runtime's site-packages layout can be located on + // disk rather than guessed. + privateWheelMutex sync.Mutex + privateWheels []string } func newSetup(path string, bp *buildplan.BuildPlan, env *envdef.Collection, depot *depot, opts *Opts) (*setup, error) { @@ -321,6 +329,12 @@ func (s *setup) update() error { return errs.Wrap(err, "errors occurred during obtain") } + // Install any decrypted private wheels now that every artifact is unpacked, + // so the Python runtime's site-packages directory can be located on disk. + if err := s.installPrivateWheels(); err != nil { + return errs.Wrap(err, "Could not install private wheels") + } + // Now we start modifying the runtime directory // This happens AFTER all the download steps are finished, and should be very fast because installing is mostly just // creating links to the depot. @@ -521,13 +535,10 @@ func (s *setup) unpack(artifact *buildplan.Artifact, b []byte) (rerr error) { } switch { case s.isPrivateWheel(unpackPath): - if err := s.installPrivateWheel(unpackPath); err != nil { - rerr := errs.Wrap(err, "Could not install private wheel") - if err2 := os.RemoveAll(unpackPath); err2 != nil { - return errs.Pack(rerr, errs.Wrap(err2, "unable to remove artifact directory")) - } - return rerr - } + // Defer installation into site-packages until all artifacts are + // unpacked; only then can we locate the Python runtime's + // site-packages layout on disk (see installPrivateWheels). + s.recordPrivateWheel(unpackPath) default: multilog.Error("Decrypted private artifact %s (%s) is of an unknown type; cannot install it", artifact.ArtifactID, artifact.Name()) } @@ -699,10 +710,44 @@ func (s *setup) isPrivateWheel(dir string) bool { return err == nil && wheelPath != "" } -// installPrivateWheel installs the decrypted wheel found under artifactDir into a -// site-packages directory and adds it to PYTHONPATH in the artifact's -// runtime.json. -func (s *setup) installPrivateWheel(artifactDir string) error { +// recordPrivateWheel notes a decrypted private wheel's depot directory for +// deferred installation into site-packages once all artifacts are unpacked. +func (s *setup) recordPrivateWheel(dir string) { + s.privateWheelMutex.Lock() + defer s.privateWheelMutex.Unlock() + s.privateWheels = append(s.privateWheels, dir) +} + +// installPrivateWheels installs each recorded private wheel into the Python +// runtime's site-packages directory. It runs after every artifact is unpacked +// so the site-packages location -- which varies by platform and Python version +// (e.g. lib/pythonX.Y/site-packages on Linux, lib/site-packages on macOS) -- can +// be discovered on disk rather than guessed. +func (s *setup) installPrivateWheels() error { + if len(s.privateWheels) == 0 { + return nil + } + + relSitePackages, err := s.locateSitePackages() + if err != nil { + return errs.Wrap(err, "could not locate the Python runtime's site-packages directory") + } + + for _, dir := range s.privateWheels { + if err := s.installPrivateWheel(dir, relSitePackages); err != nil { + return errs.Wrap(err, "could not install private wheel in %s", dir) + } + } + return nil +} + +// installPrivateWheel installs the decrypted wheel found under artifactDir into +// the runtime's site-packages directory (relSitePackages, relative to the +// install root) and adds that directory to PYTHONPATH in the artifact's +// runtime.json. Laying the wheel out at the same relative path the Python +// runtime uses means the deploy merges its contents into +// ${INSTALLDIR}/, where Python already resolves them. +func (s *setup) installPrivateWheel(artifactDir, relSitePackages string) error { wheelPath, err := findWheel(artifactDir) if err != nil { return errs.Wrap(err, "could not locate decrypted wheel") @@ -711,9 +756,16 @@ func (s *setup) installPrivateWheel(artifactDir string) error { return errs.New("decrypted private artifact contains no wheel") } - // site-packages sits in the deploy tree, so the deploy links it into the - // runtime where ${INSTALLDIR}/site-packages resolves it. - sitePackages := filepath.Join(filepath.Dir(wheelPath), "site-packages") + rtPath := filepath.Join(artifactDir, envdef.EnvironmentDefinitionFilename) + envDef, err := envdef.NewEnvironmentDefinition(rtPath) + if err != nil { + return errs.Wrap(err, "could not load runtime definition") + } + + // InstallDir is the subtree of the artifact that gets deployed onto the + // install root, so the site-packages must sit beneath it for the deploy to + // place it at ${INSTALLDIR}/. + sitePackages := filepath.Join(artifactDir, envDef.InstallDir, relSitePackages) if err := wheelinstall.Install(wheelPath, sitePackages); err != nil { return errs.Wrap(err, "could not install wheel") } @@ -721,20 +773,12 @@ func (s *setup) installPrivateWheel(artifactDir string) error { return errs.Wrap(err, "could not remove installed wheel") } - return s.exposeSitePackages(artifactDir) -} - -// exposeSitePackages adds the installed site-packages directory to PYTHONPATH in -// the artifact's runtime.json. -func (s *setup) exposeSitePackages(artifactDir string) error { - rtPath := filepath.Join(artifactDir, envdef.EnvironmentDefinitionFilename) - envDef, err := envdef.NewEnvironmentDefinition(rtPath) - if err != nil { - return errs.Wrap(err, "could not load runtime definition") - } + // Match the Python runtime's own PYTHONPATH directives (":" separator, + // inherit=false) or the cross-artifact environment merge fails with + // "incompatible separator or inherit directives". envDef.Env = append(envDef.Env, envdef.EnvironmentVariable{ Name: "PYTHONPATH", - Values: []string{"${INSTALLDIR}/site-packages"}, + Values: []string{"${INSTALLDIR}/" + filepath.ToSlash(relSitePackages)}, Join: envdef.Prepend, Inherit: false, Separator: ":", // OS-independent @@ -745,6 +789,133 @@ func (s *setup) exposeSitePackages(artifactDir string) error { return nil } +// locateSitePackages discovers the Python runtime's site-packages directory by +// scanning the unpacked non-private artifacts, and returns its path relative to +// the install root. Searching for the directory that the Python runtime already +// provides keeps us correct across platforms and Python versions instead of +// hard-coding a layout. +func (s *setup) locateSitePackages() (string, error) { + // Collect the install roots of every candidate artifact (all non-private + // artifacts that deploy something). The Python runtime is among them. + var installRoots []string + for _, artifact := range s.buildplan.Artifacts() { + artifactDir := s.depot.Path(artifact.ArtifactID) + if !fileutils.DirExists(artifactDir) { + continue + } + // Skip decrypted private artifacts: they are the wheels being installed, + // not the Python runtime that owns site-packages. + if _, info := s.depot.Exists(artifact.ArtifactID); info != nil && info.Private { + continue + } + envDef, err := envdef.NewEnvironmentDefinition(filepath.Join(artifactDir, envdef.EnvironmentDefinitionFilename)) + if err != nil { + continue // no runtime.json; nothing this artifact deploys can be site-packages + } + installRoots = append(installRoots, filepath.Join(artifactDir, envDef.InstallDir)) + } + + // First try the known layouts via cheap globs. Only if none of them match + // (an unexpected layout) do we fall back to a full walk of every root. + candidates := globSitePackages(installRoots) + if len(candidates) == 0 { + var err error + candidates, err = walkSitePackages(installRoots) + if err != nil { + return "", errs.Wrap(err, "could not scan for site-packages") + } + } + + if len(candidates) == 0 { + return "", errs.New("no Python site-packages directory found in the runtime; a Python runtime is required to install a private wheel") + } + + // Prefer the shortest path (the canonical purelib site-packages), then order + // lexicographically, so the choice is deterministic. + sort.Slice(candidates, func(i, j int) bool { + if len(candidates[i]) != len(candidates[j]) { + return len(candidates[i]) < len(candidates[j]) + } + return candidates[i] < candidates[j] + }) + if len(candidates) > 1 { + logging.Debug("Multiple site-packages directories found (%v); using %s", candidates, candidates[0]) + } + return candidates[0], nil +} + +// sitePackagesGlobs are the known relative locations of a Python runtime's +// site-packages directory, with a wildcard for the version-specific segment. +// They cover the Linux, macOS, and Windows layouts the platform produces. +var sitePackagesGlobs = []string{ + filepath.Join("lib", "python*", "site-packages"), + filepath.Join("lib64", "python*", "site-packages"), + filepath.Join("lib", "site-packages"), + filepath.Join("usr", "lib", "python*", "site-packages"), + filepath.Join("usr", "lib64", "python*", "site-packages"), + filepath.Join("Lib", "site-packages"), +} + +// globSitePackages returns the distinct site-packages directories (relative to +// their root) found under the given roots using the known glob patterns. +func globSitePackages(roots []string) []string { + var candidates []string + seen := map[string]struct{}{} + for _, root := range roots { + for _, glob := range sitePackagesGlobs { + matches, err := filepath.Glob(filepath.Join(root, glob)) + if err != nil { + continue // only ErrBadPattern, and our patterns are static + } + for _, match := range matches { + if !fileutils.DirExists(match) { + continue + } + rel, err := filepath.Rel(root, match) + if err != nil { + continue + } + if _, ok := seen[rel]; !ok { + seen[rel] = struct{}{} + candidates = append(candidates, rel) + } + } + } + } + return candidates +} + +// walkSitePackages returns the distinct site-packages directories (relative to +// their root) found by fully walking each root. It is the fallback for layouts +// the globs do not cover. +func walkSitePackages(roots []string) ([]string, error) { + var candidates []string + seen := map[string]struct{}{} + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == "site-packages" { + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if _, ok := seen[rel]; !ok { + seen[rel] = struct{}{} + candidates = append(candidates, rel) + } + return filepath.SkipDir + } + return nil + }) + if err != nil { + return nil, errs.Wrap(err, "could not scan %s for site-packages", root) + } + } + return candidates, nil +} + // findWheel returns the path of the single .whl under dir (searched recursively), // or "" if none is present. func findWheel(dir string) (string, error) { diff --git a/pkg/runtime/sitepackages_test.go b/pkg/runtime/sitepackages_test.go new file mode 100644 index 0000000000..2b6c45cd8e --- /dev/null +++ b/pkg/runtime/sitepackages_test.go @@ -0,0 +1,81 @@ +package runtime + +import ( + "os" + "path/filepath" + "testing" +) + +func mkSitePackages(t *testing.T, root, rel string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, rel), 0700); err != nil { + t.Fatalf("MkdirAll: %v", err) + } +} + +func TestGlobSitePackages(t *testing.T) { + tests := []struct { + name string + layout string // relative dir to create under the root, "" for none + want string // expected relative site-packages path, "" for no match + }{ + {"linux versioned", filepath.Join("lib", "python3.10", "site-packages"), filepath.Join("lib", "python3.10", "site-packages")}, + {"linux lib64", filepath.Join("lib64", "python3.11", "site-packages"), filepath.Join("lib64", "python3.11", "site-packages")}, + {"macos unversioned", filepath.Join("lib", "site-packages"), filepath.Join("lib", "site-packages")}, + {"usr prefixed", filepath.Join("usr", "lib", "python3.9", "site-packages"), filepath.Join("usr", "lib", "python3.9", "site-packages")}, + {"windows", filepath.Join("Lib", "site-packages"), filepath.Join("Lib", "site-packages")}, + {"no site-packages", filepath.Join("bin"), ""}, + {"unexpected layout not globbed", filepath.Join("opt", "python", "site-packages"), ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + if tt.layout != "" { + mkSitePackages(t, root, tt.layout) + } + got := globSitePackages([]string{root}) + if tt.want == "" { + if len(got) != 0 { + t.Fatalf("expected no candidates, got %v", got) + } + return + } + if len(got) != 1 || got[0] != tt.want { + t.Fatalf("expected [%s], got %v", tt.want, got) + } + }) + } +} + +func TestWalkSitePackagesFallback(t *testing.T) { + root := t.TempDir() + // A layout the globs do not cover, so only the walk finds it. + mkSitePackages(t, root, filepath.Join("opt", "python", "site-packages")) + + if got := globSitePackages([]string{root}); len(got) != 0 { + t.Fatalf("globs should not match unexpected layout, got %v", got) + } + + got, err := walkSitePackages([]string{root}) + if err != nil { + t.Fatalf("walkSitePackages: %v", err) + } + want := filepath.Join("opt", "python", "site-packages") + if len(got) != 1 || got[0] != want { + t.Fatalf("expected [%s], got %v", want, got) + } +} + +func TestGlobSitePackagesDeduplicatesAcrossRoots(t *testing.T) { + rootA := t.TempDir() + rootB := t.TempDir() + rel := filepath.Join("lib", "python3.10", "site-packages") + mkSitePackages(t, rootA, rel) + mkSitePackages(t, rootB, rel) + + got := globSitePackages([]string{rootA, rootB}) + if len(got) != 1 || got[0] != rel { + t.Fatalf("expected a single deduplicated [%s], got %v", rel, got) + } +} From c41a66db8a3e10df8235f32b527ca5e96abefa86 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 17 Jul 2026 19:47:59 -0500 Subject: [PATCH 2/4] Simplify site-packages discovery to an OS-conditional glob The Python runtime's site-packages layout is deterministic per OS, so the six-pattern glob list was overkill: Linux and macOS always use usr/lib/pythonX.Y/site-packages, and Windows always uses Lib/site-packages (case-insensitive filesystem). Collapse to a single OS-conditional pattern. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/runtime/setup.go | 55 ++++++++++++++++---------------- pkg/runtime/sitepackages_test.go | 19 +++++++---- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index 7e84be97bc..b084f26f00 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -721,8 +721,8 @@ func (s *setup) recordPrivateWheel(dir string) { // installPrivateWheels installs each recorded private wheel into the Python // runtime's site-packages directory. It runs after every artifact is unpacked // so the site-packages location -- which varies by platform and Python version -// (e.g. lib/pythonX.Y/site-packages on Linux, lib/site-packages on macOS) -- can -// be discovered on disk rather than guessed. +// (usr/lib/pythonX.Y/site-packages on Linux and macOS, Lib/site-packages on +// Windows) -- can be discovered on disk rather than guessed. func (s *setup) installPrivateWheels() error { if len(s.privateWheels) == 0 { return nil @@ -844,41 +844,40 @@ func (s *setup) locateSitePackages() (string, error) { return candidates[0], nil } -// sitePackagesGlobs are the known relative locations of a Python runtime's -// site-packages directory, with a wildcard for the version-specific segment. -// They cover the Linux, macOS, and Windows layouts the platform produces. -var sitePackagesGlobs = []string{ - filepath.Join("lib", "python*", "site-packages"), - filepath.Join("lib64", "python*", "site-packages"), - filepath.Join("lib", "site-packages"), - filepath.Join("usr", "lib", "python*", "site-packages"), - filepath.Join("usr", "lib64", "python*", "site-packages"), - filepath.Join("Lib", "site-packages"), +// sitePackagesGlob is the relative location of the Python runtime's +// site-packages directory for the current OS. Windows uses a flat, unversioned +// Lib/site-packages (the filesystem is case-insensitive, so "lib" matches too); +// every other OS uses usr/lib/pythonX.Y/site-packages, with the version segment +// left as a wildcard to be resolved on disk. +func sitePackagesGlob() string { + if sysinfo.OS() == sysinfo.Windows { + return filepath.Join("Lib", "site-packages") + } + return filepath.Join("usr", "lib", "python*", "site-packages") } // globSitePackages returns the distinct site-packages directories (relative to -// their root) found under the given roots using the known glob patterns. +// their root) found under the given roots using the OS-specific glob pattern. func globSitePackages(roots []string) []string { + glob := sitePackagesGlob() var candidates []string seen := map[string]struct{}{} for _, root := range roots { - for _, glob := range sitePackagesGlobs { - matches, err := filepath.Glob(filepath.Join(root, glob)) + matches, err := filepath.Glob(filepath.Join(root, glob)) + if err != nil { + continue // only ErrBadPattern, and our pattern is static + } + for _, match := range matches { + if !fileutils.DirExists(match) { + continue + } + rel, err := filepath.Rel(root, match) if err != nil { - continue // only ErrBadPattern, and our patterns are static + continue } - for _, match := range matches { - if !fileutils.DirExists(match) { - continue - } - rel, err := filepath.Rel(root, match) - if err != nil { - continue - } - if _, ok := seen[rel]; !ok { - seen[rel] = struct{}{} - candidates = append(candidates, rel) - } + if _, ok := seen[rel]; !ok { + seen[rel] = struct{}{} + candidates = append(candidates, rel) } } } diff --git a/pkg/runtime/sitepackages_test.go b/pkg/runtime/sitepackages_test.go index 2b6c45cd8e..b69e704550 100644 --- a/pkg/runtime/sitepackages_test.go +++ b/pkg/runtime/sitepackages_test.go @@ -4,6 +4,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/ActiveState/cli/pkg/sysinfo" ) func mkSitePackages(t *testing.T, root, rel string) { @@ -14,16 +16,18 @@ func mkSitePackages(t *testing.T, root, rel string) { } func TestGlobSitePackages(t *testing.T) { + // The expected layout is OS-specific (see sitePackagesGlob). + layout := filepath.Join("usr", "lib", "python3.10", "site-packages") + if sysinfo.OS() == sysinfo.Windows { + layout = filepath.Join("Lib", "site-packages") + } + tests := []struct { name string layout string // relative dir to create under the root, "" for none want string // expected relative site-packages path, "" for no match }{ - {"linux versioned", filepath.Join("lib", "python3.10", "site-packages"), filepath.Join("lib", "python3.10", "site-packages")}, - {"linux lib64", filepath.Join("lib64", "python3.11", "site-packages"), filepath.Join("lib64", "python3.11", "site-packages")}, - {"macos unversioned", filepath.Join("lib", "site-packages"), filepath.Join("lib", "site-packages")}, - {"usr prefixed", filepath.Join("usr", "lib", "python3.9", "site-packages"), filepath.Join("usr", "lib", "python3.9", "site-packages")}, - {"windows", filepath.Join("Lib", "site-packages"), filepath.Join("Lib", "site-packages")}, + {"matching layout", layout, layout}, {"no site-packages", filepath.Join("bin"), ""}, {"unexpected layout not globbed", filepath.Join("opt", "python", "site-packages"), ""}, } @@ -70,7 +74,10 @@ func TestWalkSitePackagesFallback(t *testing.T) { func TestGlobSitePackagesDeduplicatesAcrossRoots(t *testing.T) { rootA := t.TempDir() rootB := t.TempDir() - rel := filepath.Join("lib", "python3.10", "site-packages") + rel := filepath.Join("usr", "lib", "python3.10", "site-packages") + if sysinfo.OS() == sysinfo.Windows { + rel = filepath.Join("Lib", "site-packages") + } mkSitePackages(t, rootA, rel) mkSitePackages(t, rootB, rel) From 905451445ba39f3d8ed92ea8e15bcca057526036 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 17 Jul 2026 19:50:49 -0500 Subject: [PATCH 3/4] Drop the walk fallback for site-packages discovery If the OS-specific glob does not find the runtime's site-packages, something is wrong with the runtime -- fail fast rather than fully walking every artifact tree looking for an unexpected layout. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/runtime/setup.go | 41 -------------------------------- pkg/runtime/sitepackages_test.go | 19 --------------- 2 files changed, 60 deletions(-) diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index b084f26f00..370641d437 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -815,17 +815,7 @@ func (s *setup) locateSitePackages() (string, error) { installRoots = append(installRoots, filepath.Join(artifactDir, envDef.InstallDir)) } - // First try the known layouts via cheap globs. Only if none of them match - // (an unexpected layout) do we fall back to a full walk of every root. candidates := globSitePackages(installRoots) - if len(candidates) == 0 { - var err error - candidates, err = walkSitePackages(installRoots) - if err != nil { - return "", errs.Wrap(err, "could not scan for site-packages") - } - } - if len(candidates) == 0 { return "", errs.New("no Python site-packages directory found in the runtime; a Python runtime is required to install a private wheel") } @@ -884,37 +874,6 @@ func globSitePackages(roots []string) []string { return candidates } -// walkSitePackages returns the distinct site-packages directories (relative to -// their root) found by fully walking each root. It is the fallback for layouts -// the globs do not cover. -func walkSitePackages(roots []string) ([]string, error) { - var candidates []string - seen := map[string]struct{}{} - for _, root := range roots { - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() && d.Name() == "site-packages" { - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - if _, ok := seen[rel]; !ok { - seen[rel] = struct{}{} - candidates = append(candidates, rel) - } - return filepath.SkipDir - } - return nil - }) - if err != nil { - return nil, errs.Wrap(err, "could not scan %s for site-packages", root) - } - } - return candidates, nil -} - // findWheel returns the path of the single .whl under dir (searched recursively), // or "" if none is present. func findWheel(dir string) (string, error) { diff --git a/pkg/runtime/sitepackages_test.go b/pkg/runtime/sitepackages_test.go index b69e704550..5d926cbe6e 100644 --- a/pkg/runtime/sitepackages_test.go +++ b/pkg/runtime/sitepackages_test.go @@ -52,25 +52,6 @@ func TestGlobSitePackages(t *testing.T) { } } -func TestWalkSitePackagesFallback(t *testing.T) { - root := t.TempDir() - // A layout the globs do not cover, so only the walk finds it. - mkSitePackages(t, root, filepath.Join("opt", "python", "site-packages")) - - if got := globSitePackages([]string{root}); len(got) != 0 { - t.Fatalf("globs should not match unexpected layout, got %v", got) - } - - got, err := walkSitePackages([]string{root}) - if err != nil { - t.Fatalf("walkSitePackages: %v", err) - } - want := filepath.Join("opt", "python", "site-packages") - if len(got) != 1 || got[0] != want { - t.Fatalf("expected [%s], got %v", want, got) - } -} - func TestGlobSitePackagesDeduplicatesAcrossRoots(t *testing.T) { rootA := t.TempDir() rootB := t.TempDir() From e8954f6d2ac2a6bc01d37400306a961a60ce65c0 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Fri, 17 Jul 2026 20:02:30 -0500 Subject: [PATCH 4/4] Clean up decrypted wheel dirs when private-wheel install fails Deferring the private-wheel install lost the failure cleanup the inline path had. On failure, remove the decrypted wheel artifact directories so no private plaintext lingers and a later run re-downloads and retries (the depot rebuilds its artifact set from a disk scan, so a leftover directory would otherwise be treated as already obtained and never reinstalled). Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/runtime/setup.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/runtime/setup.go b/pkg/runtime/setup.go index 370641d437..dce980c4cc 100644 --- a/pkg/runtime/setup.go +++ b/pkg/runtime/setup.go @@ -723,11 +723,29 @@ func (s *setup) recordPrivateWheel(dir string) { // so the site-packages location -- which varies by platform and Python version // (usr/lib/pythonX.Y/site-packages on Linux and macOS, Lib/site-packages on // Windows) -- can be discovered on disk rather than guessed. -func (s *setup) installPrivateWheels() error { +func (s *setup) installPrivateWheels() (rerr error) { + // s.privateWheels is written by recordPrivateWheel from the unpack worker + // pool; this runs only after wp.Wait() has drained that pool, so the slice is + // complete and no longer written concurrently -- no lock needed here. if len(s.privateWheels) == 0 { return nil } + // On failure, remove the decrypted wheel artifact directories: this keeps no + // private plaintext on disk, and -- because the depot treats a present + // directory as already obtained -- lets a later run re-download and retry + // instead of leaving an uninstalled wheel wedged in the depot. + defer func() { + if rerr == nil { + return + } + for _, dir := range s.privateWheels { + if err := os.RemoveAll(dir); err != nil { + rerr = errs.Pack(rerr, errs.Wrap(err, "could not remove private wheel directory %s", dir)) + } + } + }() + relSitePackages, err := s.locateSitePackages() if err != nil { return errs.Wrap(err, "could not locate the Python runtime's site-packages directory")