diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go index fa5485c6..67ede89a 100644 --- a/cmd/chisel/cmd_info.go +++ b/cmd/chisel/cmd_info.go @@ -123,9 +123,12 @@ func selectPackageSlices(release *setup.Release, queries []string) (packages []* } else { releasePkg := release.Packages[pkgName] pkg = &setup.Package{ - Name: releasePkg.Name, - Archive: releasePkg.Archive, - Slices: make(map[string]*setup.Slice), + Name: releasePkg.Name, + RealName: releasePkg.RealName, + Archive: releasePkg.Archive, + Store: releasePkg.Store, + DefaultTrack: releasePkg.DefaultTrack, + Slices: make(map[string]*setup.Slice), } for _, sliceName := range pkgSlices[pkgName] { pkg.Slices[sliceName] = releasePkg.Slices[sliceName] diff --git a/internal/setup/setup.go b/internal/setup/setup.go index aaa8d1e5..67c478f6 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -16,6 +16,14 @@ import ( "github.com/canonical/chisel/internal/strdist" ) +// Store is the location from which packages are obtained via a store API. +type Store struct { + Name string + Kind string + Version string + DefaultPrefix string +} + // Release is a collection of package slices targeting a particular // distribution version. type Release struct { @@ -23,6 +31,7 @@ type Release struct { Path string Packages map[string]*Package Archives map[string]*Archive + Stores map[string]*Store Maintenance *Maintenance } @@ -51,10 +60,16 @@ type Archive struct { // Package holds a collection of slices that represent parts of themselves. type Package struct { - Name string - Path string - Archive string - Slices map[string]*Slice + // Name is the unique package identifier (e.g. "bin-curl" for store packages, + // "curl" for archive packages). + Name string + // RealName is the bare name visible in the Debian archive (e.g. "curl"). + RealName string + Path string + Archive string + Store string + DefaultTrack string + Slices map[string]*Slice } // Slice holds the details about a package slice. @@ -441,20 +456,20 @@ func readSlices(release *Release, baseDir, dirName string) error { pkgName := match[1] pkgPath := filepath.Join(dirName, entry.Name()) - if pkg, ok := release.Packages[pkgName]; ok { - return fmt.Errorf("package %q slices defined more than once: %s and %s", pkgName, pkg.Path, stripBase(baseDir, pkgPath)) - } data, err := os.ReadFile(pkgPath) if err != nil { // Errors from package os generally include the path. return fmt.Errorf("cannot read slice definition file: %v", err) } - pkg, err := parsePackage(release.Format, pkgName, stripBase(baseDir, pkgPath), data) + pkg, err := parsePackage(release, pkgName, stripBase(baseDir, pkgPath), data) if err != nil { return err } + if existing, ok := release.Packages[pkg.Name]; ok { + return fmt.Errorf("package %q slices defined more than once: %s and %s", pkg.Name, existing.Path, stripBase(baseDir, pkgPath)) + } release.Packages[pkg.Name] = pkg } return nil @@ -502,6 +517,21 @@ func Select(release *Release, slices []SliceKey, arch string) (*Selection, error new, newPath, newInfo.Generate) } } + // An invalid store kind should only throw an error if a slice references it. + // Hence, the check is here. + pkg := release.Packages[new.Package] + if pkg.Store == "" { + continue + } + store, ok := selection.Release.Stores[pkg.Store] + if !ok { + return nil, fmt.Errorf("internal error: slice %s refers to missing store %q", new, pkg.Store) + } + switch store.Kind { + case "bin": + default: + return nil, fmt.Errorf("slice %s refers to store %q with unknown kind %q", new, pkg.Store, store.Kind) + } } return selection, nil diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index d669943f..0ba3fd81 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -92,9 +92,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -139,8 +140,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -204,8 +206,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -270,9 +273,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -327,9 +331,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -582,8 +587,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -800,8 +806,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -843,8 +850,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -887,8 +895,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -960,9 +969,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -1166,8 +1176,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1241,9 +1252,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -1354,8 +1366,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "jq": { - Name: "jq", - Path: "slices/mydir/jq.yaml", + RealName: "jq", + Name: "jq", + Path: "slices/mydir/jq.yaml", Slices: map[string]*setup.Slice{ "bins": { Package: "jq", @@ -1465,8 +1478,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1538,8 +1552,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1560,8 +1575,9 @@ var setupTests = []setupTest{{ }, }, "myotherpkg": { - Name: "myotherpkg", - Path: "slices/mydir/myotherpkg.yaml", + RealName: "myotherpkg", + Name: "myotherpkg", + Path: "slices/mydir/myotherpkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "myotherpkg", @@ -1721,8 +1737,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1774,8 +1791,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1869,8 +1887,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1882,8 +1901,9 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + RealName: "mypkg2", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg2", @@ -2040,9 +2060,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2166,9 +2187,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2233,9 +2255,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2328,9 +2351,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2454,8 +2478,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg1": { - Name: "mypkg1", - Path: "slices/mydir/mypkg1.yaml", + RealName: "mypkg1", + Name: "mypkg1", + Path: "slices/mydir/mypkg1.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg1", @@ -2475,8 +2500,9 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + RealName: "mypkg2", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg2", @@ -2489,8 +2515,9 @@ var setupTests = []setupTest{{ }, }, "mypkg3": { - Name: "mypkg3", - Path: "slices/mydir/mypkg3.yaml", + RealName: "mypkg3", + Name: "mypkg3", + Path: "slices/mydir/mypkg3.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg3", @@ -2889,9 +2916,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -2939,9 +2967,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3128,9 +3157,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3250,9 +3280,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3372,9 +3403,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3494,9 +3526,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3614,9 +3647,10 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, Maintenance: &setup.Maintenance{ @@ -3657,8 +3691,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + RealName: "mypkg", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -3896,6 +3931,291 @@ var setupTests = []setupTest{{ `, }, relerror: `package "mypkg" slices defined more than once: slices/dir1/mypkg.yaml and slices/dir2/mypkg.yaml`, +}, { + summary: "Store package is parsed correctly", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + `, + }, + release: &setup.Release{ + Format: "v3", + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Maintained: true, + }, + }, + Stores: map[string]*setup.Store{ + "bin": { + Name: "bin", + Kind: "bin", + Version: "26.10", + DefaultPrefix: "bin-", + }, + }, + Packages: map[string]*setup.Package{ + "bin-mypkg": { + RealName: "mypkg", + Name: "bin-mypkg", + Path: "slices/bin/mypkg.yaml", + Store: "bin", + DefaultTrack: "3.0", + Slices: map[string]*setup.Slice{}, + }, + }, + Maintenance: &setup.Maintenance{ + Standard: time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC), + EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }, + }, +}, { + summary: "Store used with older format (v1/v2) is not allowed", + input: map[string]string{ + "chisel.yaml": strings.ReplaceAll(testutil.DefaultChiselYaml, "format: v1", "format: v2"), + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": 'store' and 'default-track' are unsupported before format v3`, +}, { + summary: "Store and archive are mutually exclusive", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + archive: ubuntu + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, +}, { + summary: "Store and archive are mutually exclusive", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + archive: ubuntu + store: bin + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": both 'store' and 'archive' fields are set`, +}, { + summary: "Store package missing default-track (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + `, + }, + relerror: `cannot parse package "mypkg": 'store' requires 'default-track'`, +}, { + summary: "default-track without store (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": 'default-track' requires 'store'`, +}, { + summary: "default-track must not contain / (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0/stable" + `, + }, + relerror: `cannot parse package "mypkg": 'default-track' must not contain /`, +}, { + summary: "Package store references undefined store (v3)", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: non-existing + default-track: "3.0" + `, + }, + relerror: `cannot parse package "mypkg": store "non-existing" not defined in release`, +}, { + summary: "Store missing version", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: store "bin" missing version field`, +}, { + summary: "Store missing kind", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + version: 26.10 + default-prefix: "bin-" + `, + }, + relerror: `chisel.yaml: store "bin" missing kind field`, +}, { + summary: "Store missing default-prefix", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + stores: + bin: + kind: bin + version: 26.10 + `, + }, + relerror: `chisel.yaml: store "bin" missing default-prefix field`, +}, { + summary: "Same-named package in archive and store", + input: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/curl.yaml": ` + package: curl + slices: + libs: + contents: + /usr/lib/libcurl.so: + bins: + contents: + /usr/bin/curl: + `, + "slices/bin/curl.yaml": ` + package: curl + store: bin + default-track: "3.0" + slices: + bins: + contents: + /usr/bin/curl-bin: + `, + }, + release: &setup.Release{ + Format: "v3", + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Maintained: true, + }, + }, + Stores: map[string]*setup.Store{ + "bin": { + Name: "bin", + Kind: "bin", + Version: "26.10", + DefaultPrefix: "bin-", + }, + }, + Packages: map[string]*setup.Package{ + "curl": { + RealName: "curl", + Name: "curl", + Path: "slices/curl.yaml", + Slices: map[string]*setup.Slice{ + "libs": { + Package: "curl", + Name: "libs", + Contents: map[string]setup.PathInfo{ + "/usr/lib/libcurl.so": {Kind: setup.CopyPath}, + }, + }, + "bins": { + Package: "curl", + Name: "bins", + Contents: map[string]setup.PathInfo{ + "/usr/bin/curl": {Kind: setup.CopyPath}, + }, + }, + }, + }, + "bin-curl": { + RealName: "curl", + Name: "bin-curl", + Path: "slices/bin/curl.yaml", + Store: "bin", + DefaultTrack: "3.0", + Slices: map[string]*setup.Slice{ + "bins": { + Package: "bin-curl", + Name: "bins", + Contents: map[string]setup.PathInfo{ + "/usr/bin/curl-bin": {Kind: setup.CopyPath}, + }, + }, + }, + }, + }, + Maintenance: &setup.Maintenance{ + Standard: time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC), + EndOfLife: time.Date(2100, time.January, 1, 0, 0, 0, 0, time.UTC), + }, + }, }} func (s *S) TestParseRelease(c *C) { @@ -3984,6 +4304,48 @@ func (s *S) TestParseRelease(c *C) { runParseReleaseTests(c, v3FormatTests) } +// TestSelectStoreUnknownKind is a dedicated test because the unknown store kind +// is only reported at selection time and only makes sense for the v3 format. +func (s *S) TestSelectStoreUnknownKind(c *C) { + runParseReleaseTests(c, []setupTest{{ + summary: "Store unknown kind", + selslices: []setup.SliceKey{{Package: "bin-mypkg", Slice: "myslice"}}, + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t\t") + ` + stores: + bin: + kind: unknown + version: 26.10 + default-prefix: "bin-" + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + slices: + myslice: + contents: + /dir/file: {} + `, + }, + selerror: `slice bin-mypkg_myslice refers to store "bin" with unknown kind "unknown"`, + }}) +} + func runParseReleaseTests(c *C, tests []setupTest) { for _, test := range tests { c.Logf("Summary: %s", test.summary) @@ -4232,6 +4594,40 @@ func (s *S) TestPackageYAMLFormat(c *C) { mypkg_three: {arch: i386} `, }, + }, { + summary: "Store package fields", + input: map[string]string{ + "chisel.yaml": ` + format: v3 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + ubuntu: + version: 26.10 + components: [main, universe] + suites: [stonking] + public-keys: [test-key] + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t\t") + ` + `, + "slices/bin/mypkg.yaml": ` + package: mypkg + store: bin + default-track: "3.0" + slices: + myslice: + contents: + /usr/bin/mypkg: {} + `, + }, }} for _, test := range tests { diff --git a/internal/setup/yaml.go b/internal/setup/yaml.go index cefe86cb..763ee781 100644 --- a/internal/setup/yaml.go +++ b/internal/setup/yaml.go @@ -35,6 +35,7 @@ type yamlRelease struct { // fields that break said compatibility (e.g. "pro" archives) and merged // together with "archives". V2Archives map[string]yamlArchive `yaml:"v2-archives"` + Stores map[string]yamlStore `yaml:"stores"` } const ( @@ -52,9 +53,17 @@ type yamlArchive struct { PubKeys []string `yaml:"public-keys"` } +type yamlStore struct { + Kind string `yaml:"kind"` + Version string `yaml:"version"` + DefaultPrefix string `yaml:"default-prefix"` +} + type yamlPackage struct { - Name string `yaml:"package"` - Archive string `yaml:"archive,omitempty"` + Name string `yaml:"package"` + Archive string `yaml:"archive,omitempty"` + Store string `yaml:"store,omitempty"` + DefaultTrack string `yaml:"default-track,omitempty"` // For backwards-compatibility reasons with v1 and v2, essential needs // custom logic to be parsed. See [yamlEssentialListMap]. Essential yamlEssentialListMap `yaml:"essential,omitempty"` @@ -433,14 +442,39 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { release.Archives[archiveName] = details } + // Parse stores. + if len(yamlVar.Stores) > 0 && (release.Format == "v1" || release.Format == "v2") { + return nil, fmt.Errorf("%s: 'stores' is unsupported before format v3", fileName) + } + if len(yamlVar.Stores) > 0 { + release.Stores = make(map[string]*Store, len(yamlVar.Stores)) + } + for storeName, details := range yamlVar.Stores { + if details.Kind == "" { + return nil, fmt.Errorf("%s: store %q missing kind field", fileName, storeName) + } + if details.Version == "" { + return nil, fmt.Errorf("%s: store %q missing version field", fileName, storeName) + } + if details.DefaultPrefix == "" { + return nil, fmt.Errorf("%s: store %q missing default-prefix field", fileName, storeName) + } + release.Stores[storeName] = &Store{ + Name: storeName, + Kind: details.Kind, + Version: details.Version, + DefaultPrefix: details.DefaultPrefix, + } + } + return release, err } -func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error) { +func parsePackage(release *Release, pkgName, pkgPath string, data []byte) (*Package, error) { pkg := Package{ - Name: pkgName, - Path: pkgPath, - Slices: make(map[string]*Slice), + RealName: pkgName, + Path: pkgPath, + Slices: make(map[string]*Slice), } yamlPkg := yamlPackage{} @@ -450,32 +484,64 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error if err != nil { return nil, fmt.Errorf("cannot parse package %q slice definitions: %v", pkgName, err) } - if yamlPkg.Name != pkg.Name { + if yamlPkg.Name != pkg.RealName { return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) } - if format == "v1" || format == "v2" { + if (yamlPkg.Store != "" || yamlPkg.DefaultTrack != "") && (release.Format == "v1" || release.Format == "v2") { + return nil, fmt.Errorf("cannot parse package %q: 'store' and 'default-track' are unsupported before format v3", pkgName) + } + if yamlPkg.Store != "" && yamlPkg.Archive != "" { + return nil, fmt.Errorf("cannot parse package %q: both 'store' and 'archive' fields are set", pkgName) + } + if yamlPkg.Store != "" { + if yamlPkg.DefaultTrack == "" { + return nil, fmt.Errorf("cannot parse package %q: 'store' requires 'default-track'", pkgName) + } + if strings.Contains(yamlPkg.DefaultTrack, "/") { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' must not contain /", pkgName) + } + pkg.Store = yamlPkg.Store + pkg.DefaultTrack = yamlPkg.DefaultTrack + } else { + if yamlPkg.DefaultTrack != "" { + return nil, fmt.Errorf("cannot parse package %q: 'default-track' requires 'store'", pkgName) + } + } + + // Derive the package unique name from its store prefix if applicable. + var prefix string + if pkg.Store != "" { + store, ok := release.Stores[pkg.Store] + if !ok { + return nil, fmt.Errorf("cannot parse package %q: store %q not defined in release", pkgName, pkg.Store) + } + prefix = store.DefaultPrefix + } + pkg.Name = prefix + pkgName + + if release.Format == "v1" || release.Format == "v2" { if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkgName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a list", pkg.Name) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != listEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a list", SliceKey{pkg.Name, sliceName}) } } } else { if yamlPkg.V3Essential != nil { - return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkgName) + return nil, fmt.Errorf("cannot parse package %q: v3-essential is obsolete since format v3", pkg.Name) } if yamlPkg.Essential.style != unsetEssential && yamlPkg.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkgName) + return nil, fmt.Errorf("cannot parse package %q: essential expects a map", pkg.Name) } for sliceName, yamlSlice := range yamlPkg.Slices { if yamlSlice.V3Essential != nil { - return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: v3-essential is obsolete since format v3", SliceKey{pkg.Name, sliceName}) } if yamlSlice.Essential.style != unsetEssential && yamlSlice.Essential.style != mapEssential { - return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{pkgName, sliceName}) + return nil, fmt.Errorf("cannot parse slice %s: essential expects a map", SliceKey{pkg.Name, sliceName}) } } } @@ -491,10 +557,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error return !unicode.IsPrint(r) }) if len(yamlSlice.Hint) > 40 || hintNotPrintable { - return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{pkgName, sliceName}, yamlSlice.Hint) + return nil, fmt.Errorf("slice %s has invalid hint %q (must be len <= 40, only contain letters, numbers, symbols and \" \")", SliceKey{pkg.Name, sliceName}, yamlSlice.Hint) } slice := &Slice{ - Package: pkgName, + Package: pkg.Name, Name: sliceName, Hint: yamlSlice.Hint, Scripts: SliceScripts{ @@ -531,17 +597,17 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error zeroPathGenerate.Generate = yamlPath.Generate if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Prefer != "" || yamlPath.Until != UntilNone { return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", - pkgName, sliceName, contPath) + pkg.Name, sliceName, contPath) } if _, err := validateGeneratePath(contPath); err != nil { - return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err) + return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkg.Name, sliceName, err) } kinds = append(kinds, GeneratePath) } else if strings.ContainsAny(contPath, "*?") { if yamlPath != nil { if !yamlPath.SameContent(&zeroPath) || yamlPath.Prefer != "" { return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", - pkgName, sliceName, contPath) + pkg.Name, sliceName, contPath) } } kinds = append(kinds, GlobPath) @@ -554,7 +620,7 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error if yamlPath.Dir { if !strings.HasSuffix(contPath, "/") { return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", - pkgName, sliceName, contPath) + pkg.Name, sliceName, contPath) } kinds = append(kinds, DirPath) } @@ -577,17 +643,17 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error switch until { case UntilNone, UntilMutate: default: - return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkgName, sliceName, contPath, until) + return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkg.Name, sliceName, contPath, until) } arch = yamlPath.Arch.List for _, s := range arch { if deb.ValidateArch(s) != nil { - return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkgName, sliceName, contPath, s) + return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkg.Name, sliceName, contPath, s) } } } - if prefer == pkgName { - return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkgName, sliceName, contPath) + if prefer == pkg.Name { + return nil, fmt.Errorf("slice %s_%s cannot 'prefer' its own package for path %s", pkg.Name, sliceName, contPath) } if len(kinds) == 0 { kinds = append(kinds, CopyPath) @@ -597,10 +663,10 @@ func parsePackage(format, pkgName, pkgPath string, data []byte) (*Package, error for i, s := range kinds { list[i] = string(s) } - return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkgName, sliceName, contPath, strings.Join(list, ", ")) + return nil, fmt.Errorf("conflict in slice %s_%s definition for path %s: %s", pkg.Name, sliceName, contPath, strings.Join(list, ", ")) } if mutable && kinds[0] != TextPath && (kinds[0] != CopyPath || isDir) { - return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath) + return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkg.Name, sliceName, contPath) } slice.Contents[contPath] = PathInfo{ Kind: kinds[0], @@ -690,9 +756,11 @@ func sliceToYAML(s *Slice) (*yamlSlice, error) { // packageToYAML converts a Package object to a yamlPackage object. func packageToYAML(p *Package) (*yamlPackage, error) { pkg := &yamlPackage{ - Name: p.Name, - Archive: p.Archive, - Slices: make(map[string]yamlSlice, len(p.Slices)), + Name: p.RealName, + Archive: p.Archive, + Store: p.Store, + DefaultTrack: p.DefaultTrack, + Slices: make(map[string]yamlSlice, len(p.Slices)), } for name, slice := range p.Slices { yamlSlice, err := sliceToYAML(slice) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb..3c5b1148 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -37,6 +37,23 @@ type pathData struct { hardLink bool } +type sourceKind int + +const ( + sourceArchive sourceKind = iota + sourceStore +) + +// pkgSourceInfo records the resolved source for a package in the selection. +// It abstracts over archive and store packages. +type pkgSourceInfo struct { + arch string + kind sourceKind + archive archive.Archive + // TODO: add store handle when store support is implemented. + pkg *setup.Package +} + type contentChecker struct { knownPaths map[string]pathData } @@ -90,7 +107,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + pkgSources, err := resolvePkgSources(options.Archives, options.Selection) if err != nil { return err } @@ -108,12 +125,12 @@ func Run(options *RunOptions) error { extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } - arch := pkgArchive[slice.Package].Options().Arch + src := pkgSources[slice.Package] for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue } - if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { continue } if preferredPkg, ok := prefers[targetPath]; ok && preferredPkg.Name != slice.Package { @@ -152,7 +169,14 @@ func Run(options *RunOptions) error { if packages[slice.Package] != nil { continue } - reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) + src := pkgSources[slice.Package] + // Store packages are distributed as "ar" archives, whose extraction is + // not yet implemented. Fail until store handling and the "ar" format + // support are in place. + if src.kind == sourceStore { + return fmt.Errorf("cannot fetch package %q from store: store packages are not yet supported", src.pkg.Name) + } + reader, info, err := src.archive.Fetch(src.pkg.RealName) if err != nil { return err } @@ -269,9 +293,9 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArchive[slice.Package].Options().Arch + src := pkgSources[slice.Package] for relPath, pathInfo := range slice.Contents { - if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, src.arch) { continue } if pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath || @@ -489,10 +513,12 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent }) } -// selectPkgArchives selects the highest priority archive containing the package -// unless a particular archive is pinned within the slice definition file. It -// returns a map of archives indexed by package names. -func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { +// resolvePkgSources determines the source for each package in the selection. +// For archive packages it selects the highest priority archive containing the +// package unless a particular archive is pinned within the slice definition +// file. For store packages it records the store reference. It returns a map +// of pkgSourceInfo indexed by package names. +func resolvePkgSources(archives map[string]archive.Archive, selection *setup.Selection) (map[string]*pkgSourceInfo, error) { sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) for _, archive := range selection.Release.Archives { if archive.Priority < 0 { @@ -506,12 +532,20 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return b.Priority - a.Priority }) - pkgArchive := make(map[string]archive.Archive) + pkgSources := make(map[string]*pkgSourceInfo) for _, s := range selection.Slices { - if _, ok := pkgArchive[s.Package]; ok { + if _, ok := pkgSources[s.Package]; ok { continue } pkg := selection.Release.Packages[s.Package] + if pkg.Store != "" { + pkgSources[pkg.Name] = &pkgSourceInfo{ + // TODO: Fill with the live store handle when store support is implemented. + kind: sourceStore, + pkg: pkg, + } + continue + } var candidates []*setup.Archive if pkg.Archive == "" { @@ -525,15 +559,38 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel var chosen archive.Archive for _, archiveInfo := range candidates { archive := archives[archiveInfo.Name] - if archive != nil && archive.Exists(pkg.Name) { + if archive != nil && archive.Exists(pkg.RealName) { chosen = archive break } } if chosen == nil { - return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) + return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) + } + pkgSources[pkg.Name] = &pkgSourceInfo{ + arch: chosen.Options().Arch, + kind: sourceArchive, + archive: chosen, + pkg: pkg, + } + } + + // Until a store is implemented as a package source there is no proper way to + // determine the architecture for store packages. + // So relying on the fact that all packages in a selection share the same architecture, + // we can borrow it from any archive package that was already resolved. + var arch string + for _, src := range pkgSources { + if src.kind == sourceArchive { + arch = src.arch + break + } + } + for _, src := range pkgSources { + if src.kind == sourceStore { + src.arch = arch } - pkgArchive[pkg.Name] = chosen } - return pkgArchive, nil + + return pkgSources, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 8d3cf775..ddfea605 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1978,6 +1978,30 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, +}, { + summary: "Store package fails as it is not yet supported", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: + `, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: bin + default-track: stable + slices: + myslice: + contents: + /dir/store-file: + `, + }, + error: `cannot fetch package "bin-store-pkg" from store: store packages are not yet supported`, }} func (s *S) TestRun(c *C) { @@ -1989,7 +2013,7 @@ func (s *S) TestRun(c *C) { for _, t := range slicerTests { m := make(map[string]string) for k, v := range t.release { - if !strings.Contains(v, "v2-archives:") { + if !strings.Contains(v, "v2-archives:") && strings.Contains(v, "format: v1") { v = strings.ReplaceAll(v, "archives:", "v2-archives:") } m[k] = v diff --git a/internal/testutil/defaults.go b/internal/testutil/defaults.go index fa08b4e6..046d4636 100644 --- a/internal/testutil/defaults.go +++ b/internal/testutil/defaults.go @@ -1,5 +1,7 @@ package testutil +import "strings" + var testKey = PGPKeys["key1"] var DefaultChiselYaml = ` @@ -17,3 +19,11 @@ var DefaultChiselYaml = ` test-key: id: ` + testKey.ID + ` armor: |` + "\n" + PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + +var DefaultChiselYamlWithStores = strings.ReplaceAll(DefaultChiselYaml, "format: v1", "format: v3") + ` + stores: + bin: + kind: bin + version: 26.10 + default-prefix: "bin-" +`