diff --git a/internal/archive/archive.go b/internal/archive/archive.go index d301494b5..74605dfbd 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -26,18 +26,19 @@ type Archive interface { } type PackageInfo struct { - Name string - Version string - Arch string - SHA256 string + Name string + Version string + Arch string + Digest string + DigestKind cache.DigestKind } func (p *PackageInfo) PkgName() string { return p.Name } func (p *PackageInfo) PkgVersion() string { return p.Version } func (p *PackageInfo) PkgRevision() int { return 0 } func (p *PackageInfo) PkgArch() string { return p.Arch } -func (p *PackageInfo) PkgDigestKind() cache.DigestKind { return cache.SHA256 } -func (p *PackageInfo) PkgDigest() string { return p.SHA256 } +func (p *PackageInfo) PkgDigestKind() cache.DigestKind { return p.DigestKind } +func (p *PackageInfo) PkgDigest() string { return p.Digest } type Options struct { Label string @@ -152,7 +153,7 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadSeekCloser, *PackageInfo, erro if err != nil { return nil, nil, err } - info := sectionPackageInfo(section) + info := sectionPackageInfo(section, digest, digestKind) return reader, info, nil } @@ -161,7 +162,8 @@ func (a *ubuntuArchive) Info(pkg string) (*PackageInfo, error) { if err != nil { return nil, err } - info := sectionPackageInfo(section) + digest, digestKind := packageDigest(section) + info := sectionPackageInfo(section, digest, digestKind) return info, nil } @@ -365,8 +367,21 @@ func findDigest(release control.Section, path string, order []digestField) (dige return "", digestField{} } +// packageDigestFields lists the checksum fields Chisel looks up in a package +// section, in order of preference: sha256 first. +var packageDigestFields = []digestField{ + {"SHA256", cache.SHA256}, + {"SHA512", cache.SHA512}, +} + +// packageDigest returns the digest recorded for the package in the section, +// along with its kind. SHA256 is preferred over stronger digests so that +// the digest used for verification, caching and the manifest keeps matching +// the one consumers expect for as long as archives publish it. Unlike index +// files, packages are fetched by their named pool path, so the by-hash +// layout does not constrain the preference order. func packageDigest(section control.Section) (digest string, kind cache.DigestKind) { - for _, f := range digestFields { + for _, f := range packageDigestFields { if d := section.Get(f.name); d != "" { return d, f.kind } @@ -516,12 +531,13 @@ func (index *ubuntuIndex) fetch(path, digest string, digestKind cache.DigestKind return index.archive.cache.Open(digestKind, writer.Digest()) } -func sectionPackageInfo(section control.Section) *PackageInfo { +func sectionPackageInfo(section control.Section, digest string, digestKind cache.DigestKind) *PackageInfo { return &PackageInfo{ - Name: section.Get("Package"), - Version: section.Get("Version"), - Arch: section.Get("Architecture"), - SHA256: section.Get("SHA256"), + Name: section.Get("Package"), + Version: section.Get("Version"), + Arch: section.Get("Architecture"), + Digest: digest, + DigestKind: digestKind, } } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b89a8d71d..660c2b6a1 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -5,7 +5,6 @@ import ( . "gopkg.in/check.v1" "crypto/sha256" - "crypto/sha512" "debug/elf" "errors" "flag" @@ -20,6 +19,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/archive/testarchive" + "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/tarball" "github.com/canonical/chisel/internal/testutil" ) @@ -252,10 +252,11 @@ func (s *httpSuite) TestFetchPackage(c *C) { pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg1", - Version: "1.1", - Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + Digest: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") @@ -263,10 +264,11 @@ func (s *httpSuite) TestFetchPackage(c *C) { pkg, info, err = testArchive.Fetch("mypkg4") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg4", - Version: "1.4", - Arch: "amd64", - SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + Name: "mypkg4", + Version: "1.4", + Arch: "amd64", + Digest: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -290,16 +292,25 @@ func (s *httpSuite) TestFetchSHA512Digests(c *C) { testArchive, err := archive.Open(&options) c.Assert(err, IsNil) - pkg, _, err := testArchive.Fetch("mypkg1") + pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) + c.Assert(info, DeepEquals, &archive.PackageInfo{ + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + Digest: "27c6e88def3d3848f4a068040bddbf908ab90e33bf93fc24fd02af7ed6a1953151302f2c59306313f065163143b51f1000cd22d102b7a58d7efd6430f5e162fb", + DigestKind: cache.SHA512, + }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") } func (s *httpSuite) TestFetchBothDigests(c *C) { // An archive publishing both SHA256 and SHA512 sections (index table and - // package fields) must be handled, with the strongest digest preferred - // for verification and caching. PackageInfo.SHA256 still surfaces: it is - // read from the package section directly, not from the preference order. + // package fields) must be handled. For packages, SHA256 is preferred for + // verification, caching and the manifest, so the recorded digest keeps + // matching the one consumers expect; the strongest digest is used only + // when the archive does not publish SHA256. Index files still use the + // strongest digest, as required by the by-hash layout. s.prepareArchiveAdjustRelease("stonking", "25.10", "amd64", []string{"main", "universe"}, []string{"SHA256", "SHA512"}, nil) @@ -319,17 +330,18 @@ func (s *httpSuite) TestFetchBothDigests(c *C) { pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg1", - Version: "1.1", - Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + Digest: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") // Pin the cache key: with both digests advertised, the package is cached - // under its strongest digest. - sha512Digest := fmt.Sprintf("%x", sha512.Sum512([]byte("mypkg1 1.1 data"))) - _, err = os.Stat(filepath.Join(options.CacheDir, "sha512", sha512Digest)) + // under its SHA256 digest. + _, err = os.Stat(filepath.Join(options.CacheDir, "sha256", + "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05")) c.Assert(err, IsNil) } @@ -356,10 +368,11 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg1", - Version: "1.1", - Arch: "arm64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Name: "mypkg1", + Version: "1.1", + Arch: "arm64", + Digest: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "mypkg1 1.1 data") @@ -367,10 +380,11 @@ func (s *httpSuite) TestFetchPortsPackage(c *C) { pkg, info, err = testArchive.Fetch("mypkg4") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg4", - Version: "1.4", - Arch: "arm64", - SHA256: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + Name: "mypkg4", + Version: "1.4", + Arch: "arm64", + Digest: "54af70097b30b33cfcbb6911ad3d0df86c2d458928169e348fa7873e4fc678e4", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "mypkg4 1.4 data") } @@ -407,20 +421,22 @@ func (s *httpSuite) TestFetchSecurityPackage(c *C) { pkg, info, err := testArchive.Fetch("mypkg1") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg1", - Version: "1.1.2.2", - Arch: "amd64", - SHA256: "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c", + Name: "mypkg1", + Version: "1.1.2.2", + Arch: "amd64", + Digest: "5448585bdd916e5023eff2bc1bc3b30bcc6ee9db9c03e531375a6a11ddf0913c", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "package from jammy-security") pkg, info, err = testArchive.Fetch("mypkg2") c.Assert(err, IsNil) c.Assert(info, DeepEquals, &archive.PackageInfo{ - Name: "mypkg2", - Version: "1.2", - Arch: "amd64", - SHA256: "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79", + Name: "mypkg2", + Version: "1.2", + Arch: "amd64", + Digest: "a4b4f3f3a8fa09b69e3ba23c60a41a1f8144691fd371a2455812572fd02e6f79", + DigestKind: cache.SHA256, }) c.Assert(read(pkg), Equals, "mypkg2 1.2 data") } @@ -662,10 +678,11 @@ var packageInfoTests = []struct { summary: "Basic", pkg: "mypkg1", info: &archive.PackageInfo{ - Name: "mypkg1", - Version: "1.1", - Arch: "amd64", - SHA256: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + Name: "mypkg1", + Version: "1.1", + Arch: "amd64", + Digest: "1f08ef04cfe7a8087ee38a1ea35fa1810246648136c3c42d5a61ad6503d85e05", + DigestKind: cache.SHA256, }, }, { summary: "Package not found in archive", diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 3b5621ecc..2f0c876c4 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "slices" "time" "golang.org/x/crypto/sha3" @@ -102,6 +103,15 @@ const ( var digestKinds = []DigestKind{SHA256, SHA384, SHA512} +// ValidateKind returns an error unless kind is a digest kind Chisel +// supports. +func ValidateKind(kind DigestKind) error { + if !slices.Contains(digestKinds, kind) { + return fmt.Errorf("unsupported digest kind: %q", kind) + } + return nil +} + var ErrMiss = fmt.Errorf("not cached") func (c *Cache) filePath(digestKind DigestKind, digest string) string { diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 3cf4db75d..e47f84a8e 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -86,11 +86,12 @@ func Write(options *WriteOptions, writer io.Writer) error { func manifestAddPackages(dbw *jsonwall.DBWriter, infos []PackageInfo) error { for _, info := range infos { err := dbw.Add(&manifest.Package{ - Kind: "package", - Name: info.PkgName(), - Version: info.PkgVersion(), - Digest: info.PkgDigest(), - Arch: info.PkgArch(), + Kind: "package", + Name: info.PkgName(), + Version: info.PkgVersion(), + Digest: info.PkgDigest(), + DigestKind: string(info.PkgDigestKind()), + Arch: info.PkgArch(), }) if err != nil { return err @@ -272,13 +273,13 @@ func validatePackage(pkg PackageInfo) (err error) { if pkg.PkgArch() == "" { return fmt.Errorf("package %q missing arch", name) } - // The manifest records the package digest as a SHA256 one. Fail rather - // than recording a digest of another kind under that name. - // TODO: record packages whose digest is not a SHA256 one, such as the - // ones coming from a store. This requires recording the digest kind in - // the manifest as well. - if pkg.PkgDigestKind() != cache.SHA256 || pkg.PkgDigest() == "" { - return fmt.Errorf("package %q missing sha256", name) + kind := pkg.PkgDigestKind() + err = cache.ValidateKind(kind) + if err != nil { + return fmt.Errorf("package %q: %s", name, err) + } + if pkg.PkgDigest() == "" { + return fmt.Errorf("package %q missing %s", name, kind) } if pkg.PkgVersion() == "" { return fmt.Errorf("package %q missing version", name) @@ -298,7 +299,27 @@ func Validate(mfest *manifest.Manifest) (err error) { pkgExist := map[string]bool{} err = mfest.IteratePackages(func(pkg *manifest.Package) error { - pkgExist[pkg.Name] = true + name := pkg.Name + if name == "" { + return fmt.Errorf("package name not set") + } + if pkg.Arch == "" { + return fmt.Errorf("package %q missing arch", name) + } + kind := cache.DigestKind(pkg.DigestKind) + if kind == "" { + return fmt.Errorf("package %q missing digest", name) + } + if err := cache.ValidateKind(kind); err != nil { + return fmt.Errorf("package %q: %s", name, err) + } + if pkg.Digest == "" { + return fmt.Errorf("package %q missing %s", name, kind) + } + if pkg.Version == "" { + return fmt.Errorf("package %q missing version", name) + } + pkgExist[name] = true return nil }) if err != nil { diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index e7e68752d..494a3c9e2 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -13,6 +13,7 @@ import ( "github.com/canonical/chisel/internal/apachetestutil" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/public/manifest" @@ -150,16 +151,18 @@ var generateManifestTests = []struct { }, packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package1", - Version: "v1", - Arch: "a1", - SHA256: "s1", + Name: "package1", + Version: "v1", + Arch: "a1", + Digest: "s1", + DigestKind: cache.SHA256, }, &archive.PackageInfo{ - Name: "package2", - Version: "v2", - Arch: "a2", - SHA256: "s2", + Name: "package2", + Version: "v2", + Arch: "a2", + Digest: "s2", + DigestKind: cache.SHA256, }, }, expected: &apachetestutil.ManifestContents{ @@ -179,17 +182,19 @@ var generateManifestTests = []struct { Slices: []string{"package1_slice1", "package2_slice2"}, }}, Packages: []*manifest.Package{{ - Kind: "package", - Name: "package1", - Version: "v1", - Digest: "s1", - Arch: "a1", + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s1", + DigestKind: "sha256", + Arch: "a1", }, { - Kind: "package", - Name: "package2", - Version: "v2", - Digest: "s2", - Arch: "a2", + Kind: "package", + Name: "package2", + Version: "v2", + Digest: "s2", + DigestKind: "sha256", + Arch: "a2", }}, Slices: []*manifest.Slice{{ Kind: "slice", @@ -212,6 +217,100 @@ var generateManifestTests = []struct { Path: "/link", }}, }, +}, { + summary: "SHA512 package digest", + selection: []*setup.Slice{slice1}, + report: &manifestutil.Report{ + Root: "/", + Entries: map[string]manifestutil.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0o456, + Slices: map[*setup.Slice]bool{slice1: true}, + }, + }, + }, + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package1", + Version: "v1", + Arch: "a1", + Digest: "s512", + DigestKind: cache.SHA512, + }, + }, + expected: &apachetestutil.ManifestContents{ + Paths: []*manifest.Path{{ + Kind: "path", + Path: "/file", + Mode: "0456", + Slices: []string{"package1_slice1"}, + }}, + Packages: []*manifest.Package{{ + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s512", + DigestKind: "sha512", + Arch: "a1", + }}, + Slices: []*manifest.Slice{{ + Kind: "slice", + Name: "package1_slice1", + }}, + Contents: []*manifest.Content{{ + Kind: "content", + Slice: "package1_slice1", + Path: "/file", + }}, + }, +}, { + summary: "SHA384 package digest", + selection: []*setup.Slice{slice1}, + report: &manifestutil.Report{ + Root: "/", + Entries: map[string]manifestutil.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0o456, + Slices: map[*setup.Slice]bool{slice1: true}, + }, + }, + }, + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package1", + Version: "v1", + Arch: "a1", + Digest: "s384", + DigestKind: cache.SHA384, + }, + }, + expected: &apachetestutil.ManifestContents{ + Paths: []*manifest.Path{{ + Kind: "path", + Path: "/file", + Mode: "0456", + Slices: []string{"package1_slice1"}, + }}, + Packages: []*manifest.Package{{ + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s384", + DigestKind: "sha384", + Arch: "a1", + }}, + Slices: []*manifest.Slice{{ + Kind: "slice", + Name: "package1_slice1", + }}, + Contents: []*manifest.Content{{ + Kind: "content", + Slice: "package1_slice1", + Path: "/file", + }}, + }, }, { summary: "Missing slice", report: &manifestutil.Report{ @@ -400,10 +499,11 @@ var generateManifestTests = []struct { }, packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package1", - Version: "v1", - Arch: "a1", - SHA256: "s1", + Name: "package1", + Version: "v1", + Arch: "a1", + Digest: "s1", + DigestKind: cache.SHA256, }, }, expected: &apachetestutil.ManifestContents{ @@ -427,11 +527,12 @@ var generateManifestTests = []struct { Inode: 1, }}, Packages: []*manifest.Package{{ - Kind: "package", - Name: "package1", - Version: "v1", - Digest: "s1", - Arch: "a1", + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s1", + DigestKind: "sha256", + Arch: "a1", }}, Slices: []*manifest.Slice{{ Kind: "slice", @@ -501,9 +602,10 @@ var generateManifestTests = []struct { summary: "Invalid package: missing name", packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Version: "v1", - Arch: "a1", - SHA256: "s1", + Version: "v1", + Arch: "a1", + Digest: "s1", + DigestKind: cache.SHA256, }, }, error: `internal error: invalid manifest: package name not set`, @@ -511,9 +613,10 @@ var generateManifestTests = []struct { summary: "Invalid package: missing version", packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package-1", - Arch: "a1", - SHA256: "s1", + Name: "package-1", + Arch: "a1", + Digest: "s1", + DigestKind: cache.SHA256, }, }, error: `internal error: invalid manifest: package "package-1" missing version`, @@ -521,22 +624,36 @@ var generateManifestTests = []struct { summary: "Invalid package: missing arch", packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package-1", - Version: "v1", - SHA256: "s1", + Name: "package-1", + Version: "v1", + Digest: "s1", + DigestKind: cache.SHA256, }, }, error: `internal error: invalid manifest: package "package-1" missing arch`, }, { - summary: "Invalid package: missing sha256", + summary: "Invalid package: missing digest", packageInfo: []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package-1", - Version: "v1", - Arch: "a1", + Name: "package-1", + Version: "v1", + Arch: "a1", + DigestKind: cache.SHA256, }, }, error: `internal error: invalid manifest: package "package-1" missing sha256`, +}, { + summary: "Invalid package: unsupported digest kind", + packageInfo: []manifestutil.PackageInfo{ + &archive.PackageInfo{ + Name: "package-1", + Version: "v1", + Arch: "a1", + Digest: "s1", + DigestKind: cache.DigestKind("md5"), + }, + }, + error: `internal error: invalid manifest: package "package-1": unsupported digest kind: "md5"`, }} func (s *S) TestGenerateManifests(c *C) { @@ -548,10 +665,11 @@ func (s *S) TestGenerateManifests(c *C) { if test.packageInfo == nil { test.packageInfo = []manifestutil.PackageInfo{ &archive.PackageInfo{ - Name: "package1", - Version: "v1", - Arch: "a1", - SHA256: "s1", + Name: "package1", + Version: "v1", + Arch: "a1", + Digest: "s1", + DigestKind: cache.SHA256, }, } } @@ -639,6 +757,36 @@ var validateManifestTests = []struct { {"kind":"slice","name":"pkg1_myslice"} `, error: `invalid manifest: content path /dir/ has no matching entry in paths`, +}, { + summary: "Package with sha512 digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","sha512":"hash1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, +}, { + summary: "Package with sha384 digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","sha384":"hash1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, +}, { + summary: "Package with missing digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, + error: `invalid manifest: package "pkg1" missing digest`, +}, { + summary: "Package with multiple digests", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":2} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","sha512":"hash2","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, + error: `invalid manifest: cannot read manifest: package "pkg1" has multiple digests recorded`, }, { summary: "Malformed jsonwall", input: ` diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d6ef9ca0d..57e0b06ff 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -859,8 +859,8 @@ var slicerTests = []slicerTest{{ "/other-file": "file 0644 fa0c9cdb {other-package_myslice}", }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", - "other-package": "other-package v3 a3 h3", + "test-package": "test-package v1 a1 sha256 h1", + "other-package": "other-package v3 a3 sha256 h3", }, }, { summary: "Pinned archive bypasses higher priority", @@ -928,7 +928,7 @@ var slicerTests = []slicerTest{{ "/file": "file 0644 fa0c9cdb {test-package_myslice}", }, manifestPkgs: map[string]string{ - "test-package": "test-package v2 a2 h2", + "test-package": "test-package v2 a2 sha256 h2", }, }, { summary: "Pinned archive does not have the package", @@ -1097,7 +1097,38 @@ var slicerTests = []slicerTest{{ "/file": "file 0644 7a3e00f5 {test-package_myslice}", }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", + "test-package": "test-package v1 a1 sha256 h1", + }, +}, { + summary: "Package with sha512 digest is recorded in the manifest", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + HashKind: "sha512", + Version: "v1", + Arch: "a1", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0o644, "./file", "from foo"), + }), + }}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + filesystem: map[string]string{ + "/file": "file 0644 7a3e00f5", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 sha512 h1", }, }, { summary: "Multiple slices of same package", @@ -1386,8 +1417,8 @@ var slicerTests = []slicerTest{{ `, }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", - "other-package": "other-package v2 a2 h2", + "test-package": "test-package v1 a1 sha256 h1", + "other-package": "other-package v2 a2 sha256 h2", }, }, { summary: "Two packages, only one is selected and recorded", @@ -1422,7 +1453,7 @@ var slicerTests = []slicerTest{{ `, }, manifestPkgs: map[string]string{ - "test-package": "test-package v1 a1 h1", + "test-package": "test-package v1 a1 sha256 h1", }, }, { summary: "Relative paths are properly trimmed during extraction", @@ -2220,7 +2251,7 @@ func treeDumpManifestPaths(mfest *manifest.Manifest) (map[string]string, error) func dumpManifestPkgs(mfest *manifest.Manifest) (map[string]string, error) { result := map[string]string{} err := mfest.IteratePackages(func(pkg *manifest.Package) error { - result[pkg.Name] = fmt.Sprintf("%s %s %s %s", pkg.Name, pkg.Version, pkg.Arch, pkg.Digest) + result[pkg.Name] = fmt.Sprintf("%s %s %s %s %s", pkg.Name, pkg.Version, pkg.Arch, pkg.DigestKind, pkg.Digest) return nil }) if err != nil { diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index d06fd1b0c..7a26e8636 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -6,6 +6,7 @@ import ( "io" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/cache" ) type TestArchive struct { @@ -14,14 +15,24 @@ type TestArchive struct { } type TestPackage struct { - Name string - Version string - Hash string + Name string + Version string + Hash string + // HashKind is the digest kind of Hash. When unset, Hash is treated as + // a sha256 digest. + HashKind string Arch string Data []byte Archives []string } +func (p *TestPackage) digestKind() cache.DigestKind { + if p.HashKind == "" { + return cache.SHA256 + } + return cache.DigestKind(p.HashKind) +} + func (a *TestArchive) Options() *archive.Options { return &a.Opts } @@ -32,10 +43,11 @@ func (a *TestArchive) Fetch(pkgName string) (io.ReadSeekCloser, *archive.Package return nil, nil, fmt.Errorf("cannot find package %q in archive", pkgName) } info := &archive.PackageInfo{ - Name: pkg.Name, - Version: pkg.Version, - SHA256: pkg.Hash, - Arch: pkg.Arch, + Name: pkg.Name, + Version: pkg.Version, + Digest: pkg.Hash, + DigestKind: pkg.digestKind(), + Arch: pkg.Arch, } return ReadSeekNopCloser(bytes.NewReader(pkg.Data)), info, nil } @@ -51,9 +63,10 @@ func (a *TestArchive) Info(pkgName string) (*archive.PackageInfo, error) { return nil, fmt.Errorf("cannot find package %q in archive", pkgName) } return &archive.PackageInfo{ - Name: pkg.Name, - Version: pkg.Version, - SHA256: pkg.Hash, - Arch: pkg.Arch, + Name: pkg.Name, + Version: pkg.Version, + Digest: pkg.Hash, + DigestKind: pkg.digestKind(), + Arch: pkg.Arch, }, nil } diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8b..0a46b23cc 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -3,6 +3,7 @@ package manifest import ( + "encoding/json" "fmt" "io" @@ -11,14 +12,99 @@ import ( const Schema = "1.0" +// Package describes a package installed in the target filesystem. type Package struct { + Kind string + Name string + Version string + Digest string + // DigestKind is the algorithm used to compute Digest, and is empty when + // no digest is recorded. + DigestKind string + Arch string +} + +// packageJSON is the JSON encoding of a Package, with the digest recorded +// under the field named after its kind. At most one of the digest fields may +// be set. +type packageJSON struct { Kind string `json:"kind"` Name string `json:"name,omitempty"` Version string `json:"version,omitempty"` - Digest string `json:"sha256,omitempty"` + SHA256 string `json:"sha256,omitempty"` + SHA384 string `json:"sha384,omitempty"` + SHA512 string `json:"sha512,omitempty"` Arch string `json:"arch,omitempty"` } +func (p *Package) MarshalJSON() ([]byte, error) { + pj := packageJSON{ + Kind: p.Kind, + Name: p.Name, + Version: p.Version, + Arch: p.Arch, + } + switch p.DigestKind { + case "": + if p.Digest != "" { + return nil, fmt.Errorf("cannot marshal package %q: digest set without a digest kind", p.Name) + } + case "sha256": + pj.SHA256 = p.Digest + case "sha384": + pj.SHA384 = p.Digest + case "sha512": + pj.SHA512 = p.Digest + default: + return nil, fmt.Errorf("cannot marshal package %q: unsupported digest kind %q", p.Name, p.DigestKind) + } + return json.Marshal(pj) +} + +func (p *Package) UnmarshalJSON(data []byte) error { + var pj packageJSON + err := json.Unmarshal(data, &pj) + if err != nil { + return err + } + digest, kind, err := pj.digest() + if err != nil { + return err + } + *p = Package{ + Kind: pj.Kind, + Name: pj.Name, + Version: pj.Version, + Digest: digest, + DigestKind: kind, + Arch: pj.Arch, + } + return nil +} + +// digest returns the package digest and its kind, as recorded in the wire +// representation. At most one digest field may be set. +func (pj *packageJSON) digest() (digest, kind string, err error) { + set := 0 + for _, entry := range []struct { + kind string + digest string + }{ + {"sha256", pj.SHA256}, + {"sha384", pj.SHA384}, + {"sha512", pj.SHA512}, + } { + if entry.digest != "" { + set++ + digest, kind = entry.digest, entry.kind + } + } + if set > 1 { + return "", "", fmt.Errorf("package %q has multiple digests recorded", pj.Name) + } + return digest, kind, nil +} + type Slice struct { Kind string `json:"kind"` Name string `json:"name,omitempty"` diff --git a/public/manifest/manifest_test.go b/public/manifest/manifest_test.go index d710e121d..f236fc511 100644 --- a/public/manifest/manifest_test.go +++ b/public/manifest/manifest_test.go @@ -3,6 +3,7 @@ package manifest_test import ( + "encoding/json" "os" "path" "slices" @@ -52,8 +53,8 @@ var readManifestTests = []struct { {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, SHA256: "", FinalSHA256: "", Size: 0x0, Link: ""}, }, Packages: []*manifest.Package{ - {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", Arch: "arch1"}, - {Kind: "package", Name: "pkg2", Version: "v2", Digest: "hash2", Arch: "arch2"}, + {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", DigestKind: "sha256", Arch: "arch1"}, + {Kind: "package", Name: "pkg2", Version: "v2", Digest: "hash2", DigestKind: "sha256", Arch: "arch2"}, }, Slices: []*manifest.Slice{ {Kind: "slice", Name: "pkg1_manifest"}, @@ -70,6 +71,35 @@ var readManifestTests = []struct { {Kind: "content", Slice: "pkg2_myotherslice", Path: "/dir/foo/bar/"}, }, }, +}, { + summary: "SHA512 package digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha512":"hash1","arch":"arch1"} + `, + mfest: &apachetestutil.ManifestContents{ + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", DigestKind: "sha512", Arch: "arch1"}, + }, + }, +}, { + summary: "SHA384 package digest", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha384":"hash1","arch":"arch1"} + `, + mfest: &apachetestutil.ManifestContents{ + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", DigestKind: "sha384", Arch: "arch1"}, + }, + }, +}, { + summary: "Multiple digests recorded", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","sha512":"hash2","arch":"arch1"} + `, + error: `cannot read manifest: package "pkg1" has multiple digests recorded`, }, { summary: "Unknown schema", input: ` @@ -108,6 +138,13 @@ func (s *S) TestManifestRead(c *C) { defer r.Close() mfest, err := manifest.Read(r) + if err == nil { + // Entry-level errors surface while iterating, as the manifest + // is not fully decoded on read. + err = mfest.IteratePackages(func(pkg *manifest.Package) error { + return nil + }) + } if test.error != "" { c.Assert(err, ErrorMatches, test.error) continue @@ -118,3 +155,75 @@ func (s *S) TestManifestRead(c *C) { } } } + +var marshalPackageTests = []struct { + summary string + pkg *manifest.Package + expected string + error string +}{{ + summary: "SHA256 digest", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digest: "hash1", + DigestKind: "sha256", + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"}`, +}, { + summary: "SHA512 digest", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digest: "hash1", + DigestKind: "sha512", + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","sha512":"hash1","arch":"arch1"}`, +}, { + summary: "No digest recorded", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Arch: "arch1", + }, + expected: `{"kind":"package","name":"pkg1","version":"v1","arch":"arch1"}`, +}, { + summary: "Digest set without a digest kind", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digest: "hash1", + Arch: "arch1", + }, + error: `json: error calling MarshalJSON for type \*manifest\.Package: cannot marshal package "pkg1": digest set without a digest kind`, +}, { + summary: "Unsupported digest kind", + pkg: &manifest.Package{ + Kind: "package", + Name: "pkg1", + Version: "v1", + Digest: "hash1", + DigestKind: "md5", + Arch: "arch1", + }, + error: `json: error calling MarshalJSON for type \*manifest\.Package: cannot marshal package "pkg1": unsupported digest kind "md5"`, +}} + +func (s *S) TestMarshalPackage(c *C) { + for _, test := range marshalPackageTests { + c.Logf("Summary: %s", test.summary) + data, err := json.Marshal(test.pkg) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + c.Assert(string(data), Equals, test.expected) + } +}