Skip to content

feat: fetch packages from store #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
upils wants to merge 6 commits into
base: slicer-source-interface
Choose a base branch
from
Draft

feat: fetch packages from store #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
102249d
feat: fetch from the bin store
upils Jun 19, 2026
91648af
fix: adapt to reworked slicer fetcher interface
upils Sep 3, 2026
571c301
refactor: adapt to PackageInfo interface
upils Sep 3, 2026
3399777
style: cleaning
upils Sep 3, 2026
e9d5cc5
test: improve deb/bin package handling in tests
upils Sep 3, 2026
39c42e4
fix: repair test broken by mistake
upils Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions cmd/chisel/cmd_cut.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"errors"
"fmt"
"slices"
"time"
Expand All @@ -11,6 +12,7 @@ import (
"github.com/canonical/chisel/internal/cache"
"github.com/canonical/chisel/internal/setup"
"github.com/canonical/chisel/internal/slicer"
"github.com/canonical/chisel/internal/store"
)

var shortCutHelp = "Cut a tree with selected slices"
Expand Down Expand Up @@ -121,9 +123,29 @@ func (cmd *cmdCut) Execute(args []string) error {
}
}

stores := make(map[string]store.Store)
for storeName, storeInfo := range release.Stores {
openStore, err := store.Open(&store.Options{
Arch: cmd.Arch,
CacheDir: cache.DefaultDir("chisel"),
Kind: storeInfo.Kind,
Version: storeInfo.Version,
})
if err != nil {
var unknownStoreKindError *store.UnknownStoreKindError
if errors.As(err, &unknownStoreKindError) {
logf("Store %q ignored: %v", storeName, err)
continue
}
return err
}
stores[storeName] = openStore
}

err = slicer.Run(&slicer.RunOptions{
Selection: selection,
Archives: archives,
Stores: stores,
TargetDir: cmd.RootDir,
})
return err
Expand Down
28 changes: 14 additions & 14 deletions cmd/chisel/cmd_debug_check_release_archives_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type checkReleaseArchivesTest struct {
summary string
arch string
release map[string]string
pkgs []*testutil.TestPackage
pkgs []*testutil.DebPackage
stdout string
err string
}
Expand All @@ -41,7 +41,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
contents:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./dir/"),
Expand Down Expand Up @@ -74,7 +74,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/link/b-bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./mode/"),
Expand Down Expand Up @@ -135,7 +135,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/linkbar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./mode/"),
Expand Down Expand Up @@ -168,7 +168,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/dir/bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./dir/"),
Expand Down Expand Up @@ -219,15 +219,15 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
`,
},
arch: "arm64",
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Arch: "arm64",
Info: archive.PackageInfo{Arch: "arm64"},
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./dir/"),
}),
}, {
Name: "pkg-b",
Arch: "arm64",
Info: archive.PackageInfo{Arch: "arm64"},
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0756, "./dir/"),
}),
Expand Down Expand Up @@ -266,7 +266,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/mode/bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./mode/"),
Expand Down Expand Up @@ -304,7 +304,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/mode/bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./mode/"),
Expand Down Expand Up @@ -357,7 +357,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/mode/bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./mode/"),
Expand Down Expand Up @@ -390,7 +390,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/link/bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Lnk(0777, "./link", "./one"),
Expand Down Expand Up @@ -432,7 +432,7 @@ var checkReleaseArchivesTests = []checkReleaseArchivesTest{{
/mode/bar:
`,
},
pkgs: []*testutil.TestPackage{{
pkgs: []*testutil.DebPackage{{
Name: "pkg-a",
Data: testutil.MustMakeDeb([]testutil.TarEntry{
testutil.Dir(0755, "./mode/"),
Expand Down Expand Up @@ -487,7 +487,7 @@ func (s *ChiselSuite) TestRun(c *C) {

archives := map[string]archive.Archive{}
for name, setupArchive := range release.Archives {
pkgs := make(map[string]*testutil.TestPackage)
pkgs := make(map[string]*testutil.DebPackage)
for _, pkg := range test.pkgs {
if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) {
pkgs[pkg.Name] = pkg
Expand Down
2 changes: 2 additions & 0 deletions cmd/chisel/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/canonical/chisel/internal/deb"
"github.com/canonical/chisel/internal/setup"
"github.com/canonical/chisel/internal/slicer"
"github.com/canonical/chisel/internal/store"
"github.com/canonical/chisel/internal/tarball"
//"github.com/canonical/chisel/internal/logger"
)
Expand Down Expand Up @@ -328,6 +329,7 @@ func run() error {
deb.SetLogger(log.Default())
setup.SetLogger(log.Default())
slicer.SetLogger(log.Default())
store.SetLogger(log.Default())
tarball.SetLogger(log.Default())
SetLogger(log.Default())

Expand Down
29 changes: 20 additions & 9 deletions internal/slicer/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/canonical/chisel/internal/archive"
"github.com/canonical/chisel/internal/manifestutil"
"github.com/canonical/chisel/internal/setup"
"github.com/canonical/chisel/internal/store"
)

// Fetcher fetches a package from the location selected for it in the
Expand Down Expand Up @@ -38,17 +39,18 @@ func (d *debFetcher) Fetch() (io.ReadSeekCloser, manifestutil.PackageInfo, error

// binFetcher fetches bin packages from a store.
type binFetcher struct {
arch string
name string
store string
store store.Store
track string
risk string
}

func (b *binFetcher) Arch() string {
return b.arch
return b.store.Options().Arch
}

func (b *binFetcher) Fetch() (io.ReadSeekCloser, manifestutil.PackageInfo, error) {
return nil, nil, fmt.Errorf("cannot fetch package %q from store %q: not implemented", b.name, b.store)
return b.store.Fetch(b.name, b.track, b.risk)
}

// selectPkgFetchers determines the fetcher for each package in the selection.
Expand All @@ -57,7 +59,7 @@ func (b *binFetcher) Fetch() (io.ReadSeekCloser, manifestutil.PackageInfo, error
// package slices file. For packages from a store it selects the store
// named in the package slices file. It returns a map of Fetcher indexed
// by package names.
func selectPkgFetchers(archives map[string]archive.Archive, selection *setup.Selection) (map[string]Fetcher, error) {
func selectPkgFetchers(archives map[string]archive.Archive, stores map[string]store.Store, selection *setup.Selection) (map[string]Fetcher, error) {
sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives))
for _, archive := range selection.Release.Archives {
if archive.Priority < 0 {
Expand All @@ -78,11 +80,20 @@ func selectPkgFetchers(archives map[string]archive.Archive, selection *setup.Sel
}
pkg := selection.Release.Packages[s.Package]
if pkg.Store != "" {
storeHandle := stores[pkg.Store]
if storeHandle == nil {
return nil, fmt.Errorf("internal error: no store handle for store %q", pkg.Store)
}

fetchers[pkg.Name] = &binFetcher{
name: pkg.Name,
store: pkg.Store,
// TODO: populate arch, track and risk when implementing
// fetching from the store.
name: pkg.RealName,
store: storeHandle,
// The store channel track is "<default-track>-<store version>",
// e.g. "3.1-26.10". The version pins the release series.
track: pkg.DefaultTrack + "-" + storeHandle.Options().Version,
// TODO: Risk is left empty for now; the store applies its default.
// In the future the risk will optionnaly come from the CLI.
risk: "",
}
continue
}
Expand Down
11 changes: 10 additions & 1 deletion internal/slicer/slicer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/canonical/chisel/internal/manifestutil"
"github.com/canonical/chisel/internal/scripts"
"github.com/canonical/chisel/internal/setup"
"github.com/canonical/chisel/internal/store"
"github.com/canonical/chisel/internal/tarball"
)

Expand All @@ -28,6 +29,7 @@ const manifestMode fs.FileMode = 0644
type RunOptions struct {
Selection *setup.Selection
Archives map[string]archive.Archive
Stores map[string]store.Store

@letFunny letFunny Jun 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per the comments in the other PR, this distinction makes sense and makes the code simpler in a way. Normally in Go you don't define generics to abstract over a type (i.e. a store) but to abstract over an operation, in this case there are clearly two:

  • How to fetch a package.
  • Record information from the package in manifest.

As we discussed in the previous PR, this was contemplated but it turned out to be much more complex AFAIK. I also see the difficulty in Go where the type system is not powerful so it is hard to model fetching where archive.fetch(name) takes only one args vs three in store.fetch(name, track, risk) for example.

@upils upils Jun 26, 2026 •
edited
Loading

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We agree. I could have reworked the Archive interface to have something more generic, used for archives and stores, but in reality they are different enough that 2 different interfaces made more sense.

As I go through the complete implementation I may discover that this new interface must be tweaked.

TargetDir string
}

Expand Down Expand Up @@ -90,7 +92,7 @@ func Run(options *RunOptions) error {
targetDir = filepath.Join(dir, targetDir)
}

pkgFetchers, err := selectPkgFetchers(options.Archives, options.Selection)
pkgFetchers, err := selectPkgFetchers(options.Archives, options.Stores, options.Selection)
if err != nil {
return err
}
Expand Down Expand Up @@ -239,6 +241,13 @@ func Run(options *RunOptions) error {
if reader == nil {
continue
}
pkg := options.Selection.Release.Packages[slice.Package]
// Store packages are distributed as XZ-compress tarballs, whose
// extraction is not yet implemented. Fail until the format support
// is in place.
if pkg.Store != "" {
return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", pkg.RealName)
}
err := tarball.Extract(reader, &tarball.ExtractOptions{
Package: slice.Package,
Extract: extract[slice.Package],
Expand Down
Loading
Loading