From 1c49ed77c908e549ded3f514e15f4492d8c0cc84 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Mon, 13 May 2024 10:08:58 +0600 Subject: [PATCH 01/60] feat: support multiple archives with "priority" field This commit adds support for fetching packages from multiple archives. It introduces a new field "archives..priority" which takes in an integer and specifies the priority of a certain archive. This value is considered when there are multiple archives. A package is fetched from the archive with highest priority, unless the slice definition file of that package specifies a particular archive using the "archive" field. TODO: a. Unit tests for cmd_cut.go, focusing on the archive selection for each package ("selectPkgArchive"). The PR for Chisel DB is moving code around and it seems best at the moment to wait for that change to be somewhat finalized. Reference: Specification RK018. --- cmd/chisel/cmd_cut.go | 49 ++++++++++++++++++++-- internal/setup/setup.go | 6 +-- internal/setup/setup_test.go | 74 +++++++++++++++------------------- internal/slicer/slicer.go | 17 ++------ internal/slicer/slicer_test.go | 26 ++++++++++-- 5 files changed, 109 insertions(+), 63 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 4224ed31..3541d881 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -99,14 +99,57 @@ func (cmd *cmdCut) Execute(args []string) error { archives[archiveName] = openArchive } + pkgArchives, err := selectPkgArchives(archives, selection) + if err != nil { + return err + } + _, err = slicer.Run(&slicer.RunOptions{ - Selection: selection, - Archives: archives, - TargetDir: cmd.RootDir, + Selection: selection, + PkgArchives: pkgArchives, + TargetDir: cmd.RootDir, }) return err } +// selectPkgArchives selects the appropriate archive for each selected slice +// package. It returns a map of archives indexed by package names. +func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { + pkgArchives := make(map[string]archive.Archive) + for _, s := range selection.Slices { + pkg := selection.Release.Packages[s.Package] + if _, ok := pkgArchives[pkg.Name]; ok { + continue + } + if pkg.Archive == "" { + var chosen *setup.Archive + for _, releaseArchive := range selection.Release.Archives { + archive := archives[releaseArchive.Name] + if archive == nil || !archive.Exists(pkg.Name) { + continue + } + if chosen == nil || chosen.Priority < releaseArchive.Priority { + chosen = releaseArchive + } + } + if chosen == nil { + return nil, fmt.Errorf("slice package %q missing from archive(s)", pkg.Name) + } + pkgArchives[pkg.Name] = archives[chosen.Name] + } else { + archive := archives[pkg.Archive] + if archive == nil { + return nil, fmt.Errorf("archive %q not defined", pkg.Archive) + } + if !archive.Exists(pkg.Name) { + return nil, fmt.Errorf("slice package %q missing from archive", pkg.Name) + } + pkgArchives[pkg.Name] = archive + } + } + return pkgArchives, nil +} + // TODO These need testing, and maybe moving into a common file. var releaseExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})-([0-9]+(?:\.?[0-9])+)$`) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 3d4bcb79..10fadcec 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -32,6 +32,7 @@ type Archive struct { Version string Suites []string Components []string + Priority int32 PubKeys []*packet.PublicKey } @@ -314,9 +315,6 @@ func readSlices(release *Release, baseDir, dirName string) error { if err != nil { return err } - if pkg.Archive == "" { - pkg.Archive = release.DefaultArchive - } release.Packages[pkg.Name] = pkg } @@ -336,6 +334,7 @@ type yamlArchive struct { Suites []string `yaml:"suites"` Components []string `yaml:"components"` Default bool `yaml:"default"` + Priority int32 `yaml:"priority"` PubKeys []string `yaml:"public-keys"` // V1PubKeys is used for compatibility with format "chisel-v1". V1PubKeys []string `yaml:"v1-public-keys"` @@ -496,6 +495,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Version: details.Version, Suites: details.Suites, Components: details.Components, + Priority: details.Priority, PubKeys: archiveKeys, } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 41917dfa..b08b3d66 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -85,10 +85,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, @@ -129,9 +128,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -191,9 +189,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -452,9 +449,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -667,9 +663,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -707,9 +702,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -748,9 +742,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -775,11 +768,13 @@ var setupTests = []setupTest{{ components: [main, universe] suites: [jammy] default: true + priority: 20 v1-public-keys: [test-key] bar: version: 22.04 components: [universe] suites: [jammy-updates] + priority: 10 v1-public-keys: [test-key] v1-public-keys: test-key: @@ -788,6 +783,7 @@ var setupTests = []setupTest{{ `, "slices/mydir/mypkg.yaml": ` package: mypkg + archive: foo `, }, release: &setup.Release{ @@ -799,6 +795,7 @@ var setupTests = []setupTest{{ Version: "22.04", Suites: []string{"jammy"}, Components: []string{"main", "universe"}, + Priority: 20, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, "bar": { @@ -806,6 +803,7 @@ var setupTests = []setupTest{{ Version: "22.04", Suites: []string{"jammy-updates"}, Components: []string{"universe"}, + Priority: 10, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, }, @@ -861,9 +859,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -926,10 +923,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "foo", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, @@ -1040,9 +1036,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "jq": { - Archive: "ubuntu", - Name: "jq", - Path: "slices/mydir/jq.yaml", + Name: "jq", + Path: "slices/mydir/jq.yaml", Slices: map[string]*setup.Slice{ "bins": { Package: "jq", @@ -1093,9 +1088,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1163,9 +1157,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1186,9 +1179,8 @@ var setupTests = []setupTest{{ }, }, "myotherpkg": { - Archive: "ubuntu", - Name: "myotherpkg", - Path: "slices/mydir/myotherpkg.yaml", + Name: "myotherpkg", + Path: "slices/mydir/myotherpkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "myotherpkg", diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index a7dd4520..39697210 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -18,9 +18,9 @@ import ( ) type RunOptions struct { - Selection *setup.Selection - Archives map[string]archive.Archive - TargetDir string + Selection *setup.Selection + PkgArchives map[string]archive.Archive + TargetDir string } func Run(options *RunOptions) (*Report, error) { @@ -60,7 +60,6 @@ func Run(options *RunOptions) (*Report, error) { syscall.Umask(oldUmask) }() - release := options.Selection.Release targetDir := filepath.Clean(options.TargetDir) targetDirAbs := targetDir if !filepath.IsAbs(targetDirAbs) { @@ -75,15 +74,7 @@ func Run(options *RunOptions) (*Report, error) { for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { - archiveName := release.Packages[slice.Package].Archive - archive := options.Archives[archiveName] - if archive == nil { - return nil, fmt.Errorf("archive %q not defined", archiveName) - } - if !archive.Exists(slice.Package) { - return nil, fmt.Errorf("slice package %q missing from archive", slice.Package) - } - archives[slice.Package] = archive + archives[slice.Package] = options.PkgArchives[slice.Package] extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 75099f84..6a88779b 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -893,11 +893,31 @@ func runSlicerTests(c *C, tests []slicerTest) { archives[name] = archive } + pkgArchives := map[string]archive.Archive{} + for _, pkg := range release.Packages { + if pkg.Archive == "" { + var chosen *setup.Archive + for _, releaseArchive := range selection.Release.Archives { + archive := archives[releaseArchive.Name] + if archive == nil || !archive.Exists(pkg.Name) { + continue + } + if chosen == nil || chosen.Priority < releaseArchive.Priority { + chosen = releaseArchive + } + } + c.Assert(chosen, NotNil) + pkgArchives[pkg.Name] = archives[chosen.Name] + } else { + pkgArchives[pkg.Name] = archives[pkg.Archive] + } + } + targetDir := c.MkDir() options := slicer.RunOptions{ - Selection: selection, - Archives: archives, - TargetDir: targetDir, + Selection: selection, + PkgArchives: pkgArchives, + TargetDir: targetDir, } if test.hackopt != nil { test.hackopt(c, &options) From fc452676ebc2c6eb3c2ffb9c47304498ae9f4db1 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 29 May 2024 03:32:59 +0600 Subject: [PATCH 02/60] feat: support "pro" keyword in archives In chisel.yaml, archive definitions can now use the "pro" value to specify Ubuntu Pro archives. The ``archives..pro`` value currently accepts the following values: "fips", "fips-updates", "apps" and "infra". Any other values are ignored (with a log message) and reset to "" (empty). The following table shows the base repository URL for each of the supported pro values: fips https://esm.ubuntu.com/fips/ubuntu fips-updates https://esm.ubuntu.com/fips-updates/ubuntu apps https://esm.ubuntu.com/apps/ubuntu infra https://esm.ubuntu.com/infra/ubuntu The actual mechanism for fetching packages from these archives are not in the scope of this commit and will follow in a later commit. While using multiple archives, the "priority" values of these Ubuntu Pro archives should be consistent with the following table: fips 20 fips-updates 21 apps 16 infra 15 Note that chisel does not cross-check nor ensure the consistency of "priority" with "pro" values. Reference: Specification RK018. --- internal/setup/setup.go | 19 ++++++ internal/setup/setup_test.go | 120 +++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 10fadcec..ccb6f03b 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -32,6 +32,7 @@ type Archive struct { Version string Suites []string Components []string + Pro string Priority int32 PubKeys []*packet.PublicKey } @@ -329,12 +330,23 @@ type yamlRelease struct { V1PubKeys map[string]yamlPubKey `yaml:"v1-public-keys"` } +type proValue string + +const ( + proNone proValue = "" + proFIPS proValue = "fips" + proFIPSUpdates proValue = "fips-updates" + proApps proValue = "apps" + proInfra proValue = "infra" +) + type yamlArchive struct { Version string `yaml:"version"` Suites []string `yaml:"suites"` Components []string `yaml:"components"` Default bool `yaml:"default"` Priority int32 `yaml:"priority"` + Pro proValue `yaml:"pro"` PubKeys []string `yaml:"public-keys"` // V1PubKeys is used for compatibility with format "chisel-v1". V1PubKeys []string `yaml:"v1-public-keys"` @@ -475,6 +487,12 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if details.Default { release.DefaultArchive = archiveName } + switch details.Pro { + case proNone, proApps, proFIPS, proFIPSUpdates, proInfra: + default: + logf("%s: ignored invalid pro value %q of archive %q", fileName, details.Pro, archiveName) + details.Pro = proNone + } if len(details.PubKeys) == 0 { if yamlVar.Format == "chisel-v1" { return nil, fmt.Errorf("%s: archive %q missing v1-public-keys field", fileName, archiveName) @@ -495,6 +513,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Version: details.Version, Suites: details.Suites, Components: details.Components, + Pro: string(details.Pro), Priority: details.Priority, PubKeys: archiveKeys, } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index b08b3d66..1bc7eec1 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1287,6 +1287,126 @@ var setupTests = []setupTest{{ `, }, relerror: `package "mypkg" has invalid essential slice reference: "mypkg-slice"`, +}, { + summary: "Pro values in archives", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + ubuntu: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + v1-public-keys: [test-key] + fips: + version: 20.04 + components: [main] + suites: [focal] + pro: fips + priority: 20 + v1-public-keys: [test-key] + fips-updates: + version: 20.04 + components: [main] + suites: [focal-updates] + pro: fips-updates + priority: 21 + v1-public-keys: [test-key] + apps: + version: 20.04 + components: [main] + suites: [focal-apps-security] + pro: apps + priority: 16 + v1-public-keys: [test-key] + infra: + version: 20.04 + components: [main] + suites: [focal-infra-security] + pro: infra + priority: 15 + v1-public-keys: [test-key] + foo: + version: 20.04 + components: [main] + suites: [foo] + pro: foo + priority: -10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Priority: 10, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips": { + Name: "fips", + Version: "20.04", + Suites: []string{"focal"}, + Components: []string{"main"}, + Pro: "fips", + Priority: 20, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "fips-updates": { + Name: "fips-updates", + Version: "20.04", + Suites: []string{"focal-updates"}, + Components: []string{"main"}, + Pro: "fips-updates", + Priority: 21, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "apps": { + Name: "apps", + Version: "20.04", + Suites: []string{"focal-apps-security"}, + Components: []string{"main"}, + Pro: "apps", + Priority: 16, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "infra": { + Name: "infra", + Version: "20.04", + Suites: []string{"focal-infra-security"}, + Components: []string{"main"}, + Pro: "infra", + Priority: 15, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + "foo": { + Name: "foo", + Version: "20.04", + Suites: []string{"foo"}, + Components: []string{"main"}, + // Pro value "foo" is ignored and set to "". + Priority: -10, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }} var defaultChiselYaml = ` From 0099462895932537bf272aa1d158acf2d56b546d Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 29 May 2024 17:44:46 +0600 Subject: [PATCH 03/60] feat: support fetching packages from Pro archives This commit adds the support for fetching packages from the Ubuntu Pro archives. As stated in previous commit(s), Chisel will currently support fetching from the following Ubuntu Pro archives: fips https://esm.ubuntu.com/fips/ubuntu/ fips-updates https://esm.ubuntu.com/fips-updates/ubuntu/ apps https://esm.ubuntu.com/apps/ubuntu/ infra https://esm.ubuntu.com/infra/ubuntu/ By default, Chisel will look for credentials in the ``/etc/apt/auth.conf.d/`` directory, unless the environment variable ``CHISEL_AUTH_DIR`` is set. In which case, it will look for configuration files in that directory. The configuration files may only have the ".conf" extensions or no extensions. The format of these configuration files with further details have been introduced in commit 86bcf61 previously. In short, a configuration file may look like the following: machine esm.ubuntu.com/apps/ubuntu/ login password machine esm.ubuntu.com/fips-updates/ubuntu/ login password Note that any Pro archive whose credentials are not found will be ignored (with logs). A sample chisel.yaml with different Pro archives may look like below: format: v1 archives: ubuntu: priority: 10 version: 22.04 suites: [jammy, jammy-updates, jammy-security] components: [main, universe] public-keys: [ubuntu-archive-key-2018] default: true ubuntu-fips: pro: fips priority: 20 version: 22.04 suites: [jammy] components: [main] public-keys: [ubuntu-archive-key-2018, fips-key] ubuntu-fips-updates: pro: fips-updates priority: 21 version: 22.04 suites: [jammy-updates] components: [main] public-keys: [ubuntu-archive-key-2018, fips-key] ubuntu-esm-infra: pro: infra priority: 15 version: 22.04 suites: [jammy-infra-updates, jammy-infra-security] components: [main] public-keys: [ubuntu-archive-key-2018, esm-key] ubuntu-esm-apps: pro: apps priority: 16 version: 22.04 suites: [jammy-apps-updates, jammy-apps-security] components: [main] public-keys: [ubuntu-archive-key-2018, esm-key] public-keys: ... Reference: Specification RK018. --- cmd/chisel/cmd_cut.go | 5 ++ cmd/chisel/main.go | 1 + internal/archive/archive.go | 79 ++++++++++++++++++++++++++++---- internal/archive/archive_test.go | 71 ++++++++++++++++++++++------ internal/setup/setup.go | 4 +- internal/setup/setup_test.go | 9 ---- internal/slicer/slicer_test.go | 63 +++++++++++++++++++++++++ 7 files changed, 199 insertions(+), 33 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 3541d881..c16bd35b 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -90,10 +90,15 @@ func (cmd *cmdCut) Execute(args []string) error { Arch: cmd.Arch, Suites: archiveInfo.Suites, Components: archiveInfo.Components, + Pro: archiveInfo.Pro, CacheDir: cache.DefaultDir("chisel"), PubKeys: archiveInfo.PubKeys, }) if err != nil { + if err == archive.ErrCredentialsNotFound { + logf("Ignoring archive %q (credentials not found)...", archiveName) + continue + } return err } archives[archiveName] = openArchive diff --git a/cmd/chisel/main.go b/cmd/chisel/main.go index e8f925a2..50792839 100644 --- a/cmd/chisel/main.go +++ b/cmd/chisel/main.go @@ -327,6 +327,7 @@ func run() error { deb.SetLogger(log.Default()) setup.SetLogger(log.Default()) slicer.SetLogger(log.Default()) + SetLogger(log.Default()) parser := Parser() xtra, err := parser.Parse() diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 2a552084..2637e628 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -28,6 +28,7 @@ type Options struct { Arch string Suites []string Components []string + Pro string CacheDir string PubKeys []*packet.PublicKey } @@ -42,6 +43,12 @@ func Open(options *Options) (Archive, error) { if err != nil { return nil, err } + if options.Pro != "" { + err = validatePro(options.Pro) + if err != nil { + return nil, err + } + } return openUbuntu(options) } @@ -69,6 +76,8 @@ type ubuntuArchive struct { indexes []*ubuntuIndex cache *cache.Cache pubKeys []*packet.PublicKey + baseURL string + creds *credentials } type ubuntuIndex struct { @@ -129,6 +138,44 @@ func (a *ubuntuArchive) Fetch(pkg string) (io.ReadCloser, error) { const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" +var proArchiveInfo = map[string]struct { + baseURL, label string +}{ + "fips": { + baseURL: "https://esm.ubuntu.com/fips/ubuntu/", + label: "UbuntuFIPS", + }, + "fips-updates": { + baseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", + label: "UbuntuFIPSUpdates", + }, + "apps": { + baseURL: "https://esm.ubuntu.com/apps/ubuntu/", + label: "UbuntuESMApps", + }, + "infra": { + baseURL: "https://esm.ubuntu.com/infra/ubuntu/", + label: "UbuntuESM", + }, +} + +func archiveURL(pro, arch string) string { + if pro == "" { + if arch == "amd64" || arch == "i386" { + return ubuntuURL + } + return ubuntuPortsURL + } + return proArchiveInfo[pro].baseURL +} + +func validatePro(pro string) error { + if _, ok := proArchiveInfo[pro]; !ok { + return fmt.Errorf("unknown pro value: %s", pro) + } + return nil +} + func openUbuntu(options *Options) (Archive, error) { if len(options.Components) == 0 { return nil, fmt.Errorf("archive options missing components") @@ -148,6 +195,18 @@ func openUbuntu(options *Options) (Archive, error) { pubKeys: options.PubKeys, } + if archive.baseURL == "" { + archive.baseURL = archiveURL(options.Pro, options.Arch) + } + + if options.Pro != "" && archive.creds == nil { + creds, err := findCredentials(archive.baseURL) + if err != nil { + return nil, err + } + archive.creds = creds + } + for _, suite := range options.Suites { var release control.Section for _, component := range options.Components { @@ -217,12 +276,14 @@ func (index *ubuntuIndex) fetchRelease() error { if err != nil { return fmt.Errorf("cannot parse InRelease file: %v", err) } - section := ctrl.Section("Ubuntu") + // Parse the appropriate section for the type of archive. + label := "Ubuntu" + if index.archive.options.Pro != "" { + label = proArchiveInfo[index.archive.options.Pro].label + } + section := ctrl.Section(label) if section == nil { - section = ctrl.Section("UbuntuProFIPS") - if section == nil { - return fmt.Errorf("corrupted archive InRelease file: no Ubuntu section") - } + return fmt.Errorf("corrupted archive InRelease file: no %s section", label) } logf("Release date: %s", section.Get("Date")) @@ -277,10 +338,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return nil, err } - baseURL := ubuntuURL - if index.arch != "amd64" && index.arch != "i386" { - baseURL = ubuntuPortsURL - } + baseURL, creds := index.archive.baseURL, index.archive.creds var url string if strings.HasPrefix(suffix, "pool/") { @@ -293,6 +351,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea if err != nil { return nil, fmt.Errorf("cannot create HTTP request: %v", err) } + if creds != nil && !creds.Empty() { + req.SetBasicAuth(creds.Username, creds.Password) + } var resp *http.Response if flags&fetchBulk != 0 { resp, err = bulkDo(req) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index af741dbe..ad9c77bb 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -322,7 +322,7 @@ func (s *httpSuite) TestArchiveLabels(c *C) { _, err = archive.Open(&options) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("UbuntuProFIPS")) + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("ThirdParty")) options = archive.Options{ Label: "ubuntu", @@ -335,22 +335,67 @@ func (s *httpSuite) TestArchiveLabels(c *C) { } _, err = archive.Open(&options) - c.Assert(err, IsNil) + c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) +} - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("ThirdParty")) +var proArchiveInfo = map[string]struct { + baseURL, label string +}{ + "fips": { + baseURL: "https://esm.ubuntu.com/fips/ubuntu/", + label: "UbuntuFIPS", + }, + "fips-updates": { + baseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", + label: "UbuntuFIPSUpdates", + }, + "apps": { + baseURL: "https://esm.ubuntu.com/apps/ubuntu/", + label: "UbuntuESMApps", + }, + "infra": { + baseURL: "https://esm.ubuntu.com/infra/ubuntu/", + label: "UbuntuESM", + }, +} - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, +func (s *httpSuite) TestProArchives(c *C) { + setLabel := func(label string) func(*testarchive.Release) { + return func(r *testarchive.Release) { + r.Label = label + } } - _, err = archive.Open(&options) - c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "" + for _, info := range proArchiveInfo { + contents += fmt.Sprintf("machine %s login foo password bar\n", info.baseURL) + } + err := os.WriteFile(confFile, []byte(contents), 0600) + c.Assert(err, IsNil) + + for pro, info := range proArchiveInfo { + s.base = info.baseURL + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel(info.label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err = archive.Open(&options) + c.Assert(err, IsNil) + } } type verifyArchiveReleaseTest struct { diff --git a/internal/setup/setup.go b/internal/setup/setup.go index ccb6f03b..914c9606 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -490,8 +490,8 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { switch details.Pro { case proNone, proApps, proFIPS, proFIPSUpdates, proInfra: default: - logf("%s: ignored invalid pro value %q of archive %q", fileName, details.Pro, archiveName) - details.Pro = proNone + logf("%s: archive %q ignored due to invalid pro value: %s", fileName, archiveName, details.Pro) + continue } if len(details.PubKeys) == 0 { if yamlVar.Format == "chisel-v1" { diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 1bc7eec1..ad79f385 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1389,15 +1389,6 @@ var setupTests = []setupTest{{ Priority: 15, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, - "foo": { - Name: "foo", - Version: "20.04", - Suites: []string{"foo"}, - Components: []string{"main"}, - // Pro value "foo" is ignored and set to "". - Priority: -10, - PubKeys: []*packet.PublicKey{testKey.PubKey}, - }, }, Packages: map[string]*setup.Package{ "mypkg": { diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 6a88779b..fe781197 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -792,6 +792,68 @@ var slicerTests = []slicerTest{{ report: map[string]string{ "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", }, +}, { + summary: "Pro archives", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + ubuntu: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + v1-public-keys: [test-key] + fips: + version: 20.04 + components: [main] + suites: [focal] + pro: fips + priority: 20 + v1-public-keys: [test-key] + fips-updates: + version: 20.04 + components: [main] + suites: [focal-updates] + pro: fips-updates + priority: 21 + v1-public-keys: [test-key] + apps: + version: 20.04 + components: [main] + suites: [focal-apps-security] + pro: apps + priority: 16 + v1-public-keys: [test-key] + infra: + version: 20.04 + components: [main] + suites: [focal-infra-security] + pro: infra + priority: 15 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/nested/file: + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/nested/": "dir 0755", + "/dir/nested/file": "file 0644 84237a05", + }, + report: map[string]string{ + "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", + }, }} var defaultChiselYaml = ` @@ -886,6 +948,7 @@ func runSlicerTests(c *C, tests []slicerTest) { Version: setupArchive.Version, Suites: setupArchive.Suites, Components: setupArchive.Components, + Pro: setupArchive.Pro, Arch: test.arch, }, pkgs: test.pkgs, From cfe740bd4d37e47564de18789f8f8db1d60ae6ca Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 6 Aug 2024 16:05:06 +0600 Subject: [PATCH 04/60] Merge "feat/multiple-archives" into this branch This merges the changes in https://github.com/letFunny/chisel/pull/12 into this branch. --- .github/workflows/spread.yml | 2 +- cmd/chisel/cmd_cut.go | 111 +--- cmd/chisel/cmd_find.go | 136 ++++ cmd/chisel/cmd_find_test.go | 140 +++++ cmd/chisel/cmd_help.go | 2 +- cmd/chisel/export_test.go | 2 + cmd/chisel/helpers.go | 71 +++ go.mod | 2 +- internal/deb/extract.go | 103 ++- internal/deb/extract_test.go | 122 +++- internal/fsutil/create.go | 7 +- internal/fsutil/create_test.go | 48 +- internal/scripts/scripts.go | 22 +- internal/scripts/scripts_test.go | 68 ++ internal/setup/setup.go | 49 +- internal/setup/setup_test.go | 84 ++- internal/slicer/report.go | 73 ++- internal/slicer/report_test.go | 135 +++- internal/slicer/slicer.go | 438 ++++++++----- internal/slicer/slicer_test.go | 868 +++++++++++++++++++++----- internal/strdist/strdist.go | 4 +- internal/testutil/permutation.go | 34 + internal/testutil/permutation_test.go | 73 +++ internal/testutil/treedump.go | 24 + snap/snapcraft.yaml | 2 +- spread.yaml | 2 +- tests/find/task.yaml | 14 + 27 files changed, 2030 insertions(+), 606 deletions(-) create mode 100644 cmd/chisel/cmd_find.go create mode 100644 cmd/chisel/cmd_find_test.go create mode 100644 cmd/chisel/helpers.go create mode 100644 internal/testutil/permutation.go create mode 100644 internal/testutil/permutation_test.go create mode 100644 tests/find/task.yaml diff --git a/.github/workflows/spread.yml b/.github/workflows/spread.yml index 4d40b814..190a9061 100644 --- a/.github/workflows/spread.yml +++ b/.github/workflows/spread.yml @@ -29,4 +29,4 @@ jobs: - name: Build and run spread run: | (cd _spread/cmd/spread && go build) - _spread/cmd/spread/spread -v focal jammy mantic + _spread/cmd/spread/spread -v focal jammy mantic noble diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index c16bd35b..30ad4f8a 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -3,11 +3,6 @@ package main import ( "github.com/jessevdk/go-flags" - "fmt" - "os" - "regexp" - "strings" - "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" @@ -18,10 +13,13 @@ var shortCutHelp = "Cut a tree with selected slices" var longCutHelp = ` The cut command uses the provided selection of package slices to create a new filesystem tree in the root location. + +By default it fetches the slices for the same Ubuntu version as the +current host, unless the --release flag is used. ` var cutDescs = map[string]string{ - "release": "Chisel release directory", + "release": "Chisel release name or directory (e.g. ubuntu-22.04)", "root": "Root for generated content", "arch": "Package architecture", } @@ -54,25 +52,7 @@ func (cmd *cmdCut) Execute(args []string) error { sliceKeys[i] = sliceKey } - var release *setup.Release - var err error - if strings.Contains(cmd.Release, "/") { - release, err = setup.ReadRelease(cmd.Release) - } else { - var label, version string - if cmd.Release == "" { - label, version, err = readReleaseInfo() - } else { - label, version, err = parseReleaseInfo(cmd.Release) - } - if err != nil { - return err - } - release, err = setup.FetchRelease(&setup.FetchOptions{ - Label: label, - Version: version, - }) - } + release, err := obtainRelease(cmd.Release) if err != nil { return err } @@ -104,85 +84,10 @@ func (cmd *cmdCut) Execute(args []string) error { archives[archiveName] = openArchive } - pkgArchives, err := selectPkgArchives(archives, selection) - if err != nil { - return err - } - _, err = slicer.Run(&slicer.RunOptions{ - Selection: selection, - PkgArchives: pkgArchives, - TargetDir: cmd.RootDir, + Selection: selection, + Archives: archives, + TargetDir: cmd.RootDir, }) return err } - -// selectPkgArchives selects the appropriate archive for each selected slice -// package. It returns a map of archives indexed by package names. -func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { - pkgArchives := make(map[string]archive.Archive) - for _, s := range selection.Slices { - pkg := selection.Release.Packages[s.Package] - if _, ok := pkgArchives[pkg.Name]; ok { - continue - } - if pkg.Archive == "" { - var chosen *setup.Archive - for _, releaseArchive := range selection.Release.Archives { - archive := archives[releaseArchive.Name] - if archive == nil || !archive.Exists(pkg.Name) { - continue - } - if chosen == nil || chosen.Priority < releaseArchive.Priority { - chosen = releaseArchive - } - } - if chosen == nil { - return nil, fmt.Errorf("slice package %q missing from archive(s)", pkg.Name) - } - pkgArchives[pkg.Name] = archives[chosen.Name] - } else { - archive := archives[pkg.Archive] - if archive == nil { - return nil, fmt.Errorf("archive %q not defined", pkg.Archive) - } - if !archive.Exists(pkg.Name) { - return nil, fmt.Errorf("slice package %q missing from archive", pkg.Name) - } - pkgArchives[pkg.Name] = archive - } - } - return pkgArchives, nil -} - -// TODO These need testing, and maybe moving into a common file. - -var releaseExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})-([0-9]+(?:\.?[0-9])+)$`) - -func parseReleaseInfo(release string) (label, version string, err error) { - match := releaseExp.FindStringSubmatch(release) - if match == nil { - return "", "", fmt.Errorf("invalid release reference: %q", release) - } - return match[1], match[2], nil -} - -func readReleaseInfo() (label, version string, err error) { - data, err := os.ReadFile("/etc/lsb-release") - if err == nil { - const labelPrefix = "DISTRIB_ID=" - const versionPrefix = "DISTRIB_RELEASE=" - for _, line := range strings.Split(string(data), "\n") { - switch { - case strings.HasPrefix(line, labelPrefix): - label = strings.ToLower(line[len(labelPrefix):]) - case strings.HasPrefix(line, versionPrefix): - version = line[len(versionPrefix):] - } - if label != "" && version != "" { - return label, version, nil - } - } - } - return "", "", fmt.Errorf("cannot infer release via /etc/lsb-release, see the --release option") -} diff --git a/cmd/chisel/cmd_find.go b/cmd/chisel/cmd_find.go new file mode 100644 index 00000000..91c3f7f7 --- /dev/null +++ b/cmd/chisel/cmd_find.go @@ -0,0 +1,136 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "text/tabwriter" + + "github.com/jessevdk/go-flags" + + "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/strdist" +) + +var shortFindHelp = "Find existing slices" +var longFindHelp = ` +The find command queries the slice definitions for matching slices. +Globs (* and ?) are allowed in the query. + +By default it fetches the slices for the same Ubuntu version as the +current host, unless the --release flag is used. +` + +var findDescs = map[string]string{ + "release": "Chisel release name or directory (e.g. ubuntu-22.04)", +} + +type cmdFind struct { + Release string `long:"release" value-name:""` + + Positional struct { + Query []string `positional-arg-name:"" required:"yes"` + } `positional-args:"yes"` +} + +func init() { + addCommand("find", shortFindHelp, longFindHelp, func() flags.Commander { return &cmdFind{} }, findDescs, nil) +} + +func (cmd *cmdFind) Execute(args []string) error { + if len(args) > 0 { + return ErrExtraArgs + } + + release, err := obtainRelease(cmd.Release) + if err != nil { + return err + } + + slices, err := findSlices(release, cmd.Positional.Query) + if err != nil { + return err + } + if len(slices) == 0 { + fmt.Fprintf(Stderr, "No matching slices for \"%s\"\n", strings.Join(cmd.Positional.Query, " ")) + return nil + } + + w := tabWriter() + fmt.Fprintf(w, "Slice\tSummary\n") + for _, s := range slices { + fmt.Fprintf(w, "%s\t%s\n", s, "-") + } + w.Flush() + + return nil +} + +// match reports whether a slice (partially) matches the query. +func match(slice *setup.Slice, query string) bool { + var term string + switch { + case strings.HasPrefix(query, "_"): + query = strings.TrimPrefix(query, "_") + term = slice.Name + case strings.Contains(query, "_"): + term = slice.String() + default: + term = slice.Package + } + query = strings.ReplaceAll(query, "**", "⁑") + return strdist.Distance(term, query, distWithGlobs, 0) <= 1 +} + +// findSlices returns slices from the provided release that match all of the +// query strings (AND). +func findSlices(release *setup.Release, query []string) (slices []*setup.Slice, err error) { + slices = []*setup.Slice{} + for _, pkg := range release.Packages { + for _, slice := range pkg.Slices { + if slice == nil { + continue + } + allMatch := true + for _, term := range query { + if !match(slice, term) { + allMatch = false + break + } + } + if allMatch { + slices = append(slices, slice) + } + } + } + sort.Slice(slices, func(i, j int) bool { + return slices[i].String() < slices[j].String() + }) + return slices, nil +} + +func tabWriter() *tabwriter.Writer { + return tabwriter.NewWriter(Stdout, 5, 3, 2, ' ', 0) +} + +// distWithGlobs encodes the standard Levenshtein distance with support for +// "*", "?" and "**". However, because it works on runes "**" has to be encoded +// as "⁑" in the strings. +// +// Supported wildcards: +// +// ? - Any one character +// * - Any zero or more characters +// ⁑ - Any zero or more characters +func distWithGlobs(ar, br rune) strdist.Cost { + if ar == '⁑' || br == '⁑' { + return strdist.Cost{SwapAB: 0, DeleteA: 0, InsertB: 0} + } + if ar == '*' || br == '*' { + return strdist.Cost{SwapAB: 0, DeleteA: 0, InsertB: 0} + } + if ar == '?' || br == '?' { + return strdist.Cost{SwapAB: 0, DeleteA: 1, InsertB: 1} + } + return strdist.StandardCost(ar, br) +} diff --git a/cmd/chisel/cmd_find_test.go b/cmd/chisel/cmd_find_test.go new file mode 100644 index 00000000..7d7c0d7a --- /dev/null +++ b/cmd/chisel/cmd_find_test.go @@ -0,0 +1,140 @@ +package main_test + +import ( + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/testutil" + + chisel "github.com/canonical/chisel/cmd/chisel" +) + +type findTest struct { + summary string + release *setup.Release + query []string + result []*setup.Slice +} + +func makeSamplePackage(pkg string, slices []string) *setup.Package { + slicesMap := map[string]*setup.Slice{} + for _, slice := range slices { + slicesMap[slice] = &setup.Slice{ + Package: pkg, + Name: slice, + } + } + return &setup.Package{ + Name: pkg, + Path: "slices/" + pkg, + Archive: "ubuntu", + Slices: slicesMap, + } +} + +var sampleRelease = &setup.Release{ + DefaultArchive: "ubuntu", + + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy", "jammy-security"}, + Components: []string{"main", "other"}, + }, + }, + Packages: map[string]*setup.Package{ + "openjdk-8-jdk": makeSamplePackage("openjdk-8-jdk", []string{"bins", "config", "core", "libs", "utils"}), + "python3.10": makeSamplePackage("python3.10", []string{"bins", "config", "core", "libs", "utils"}), + }, +} + +var findTests = []findTest{{ + summary: "Search by package name", + release: sampleRelease, + query: []string{"python3.10"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["bins"], + sampleRelease.Packages["python3.10"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["core"], + sampleRelease.Packages["python3.10"].Slices["libs"], + sampleRelease.Packages["python3.10"].Slices["utils"], + }, +}, { + summary: "Search by slice name", + release: sampleRelease, + query: []string{"_config"}, + result: []*setup.Slice{ + sampleRelease.Packages["openjdk-8-jdk"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["config"], + }, +}, { + summary: "Slice search without leading underscore", + release: sampleRelease, + query: []string{"config"}, + result: []*setup.Slice{}, +}, { + summary: "Check distance greater than one", + release: sampleRelease, + query: []string{"python3."}, + result: []*setup.Slice{}, +}, { + summary: "Check glob matching (*)", + release: sampleRelease, + query: []string{"python3.*_bins"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["bins"], + }, +}, { + summary: "Check glob matching (?)", + release: sampleRelease, + query: []string{"python3.1?_co*"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["core"], + }, +}, { + summary: "Check no matching slice", + release: sampleRelease, + query: []string{"foo_bar"}, + result: []*setup.Slice{}, +}, { + summary: "Several terms all match", + release: sampleRelease, + query: []string{"python*", "_co*"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["core"], + }, +}, { + summary: "Distance of one in each term", + release: sampleRelease, + query: []string{"python3.1", "_lib"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["libs"], + }, +}, { + summary: "Query with underscore is matched against full name", + release: sampleRelease, + query: []string{"python3.1_libs"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["libs"], + }, +}, { + summary: "Several terms, one does not match", + release: sampleRelease, + query: []string{"python", "slice"}, + result: []*setup.Slice{}, +}} + +func (s *ChiselSuite) TestFindSlices(c *C) { + for _, test := range findTests { + c.Logf("Summary: %s", test.summary) + + for _, query := range testutil.Permutations(test.query) { + slices, err := chisel.FindSlices(test.release, query) + c.Assert(err, IsNil) + c.Assert(slices, DeepEquals, test.result) + } + } +} diff --git a/cmd/chisel/cmd_help.go b/cmd/chisel/cmd_help.go index a6314f49..a0ebd273 100644 --- a/cmd/chisel/cmd_help.go +++ b/cmd/chisel/cmd_help.go @@ -155,7 +155,7 @@ type helpCategory struct { var helpCategories = []helpCategory{{ Label: "Basic", Description: "general operations", - Commands: []string{"help", "version"}, + Commands: []string{"find", "help", "version"}, }, { Label: "Action", Description: "make things happen", diff --git a/cmd/chisel/export_test.go b/cmd/chisel/export_test.go index dd852b44..a99a7831 100644 --- a/cmd/chisel/export_test.go +++ b/cmd/chisel/export_test.go @@ -17,3 +17,5 @@ func FakeIsStdinTTY(t bool) (restore func()) { isStdinTTY = oldIsStdinTTY } } + +var FindSlices = findSlices diff --git a/cmd/chisel/helpers.go b/cmd/chisel/helpers.go new file mode 100644 index 00000000..7506888e --- /dev/null +++ b/cmd/chisel/helpers.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/canonical/chisel/internal/setup" +) + +// TODO These need testing + +var releaseExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})-([0-9]+(?:\.?[0-9])+)$`) + +func parseReleaseInfo(release string) (label, version string, err error) { + match := releaseExp.FindStringSubmatch(release) + if match == nil { + return "", "", fmt.Errorf("invalid release reference: %q", release) + } + return match[1], match[2], nil +} + +func readReleaseInfo() (label, version string, err error) { + data, err := os.ReadFile("/etc/lsb-release") + if err == nil { + const labelPrefix = "DISTRIB_ID=" + const versionPrefix = "DISTRIB_RELEASE=" + for _, line := range strings.Split(string(data), "\n") { + switch { + case strings.HasPrefix(line, labelPrefix): + label = strings.ToLower(line[len(labelPrefix):]) + case strings.HasPrefix(line, versionPrefix): + version = line[len(versionPrefix):] + } + if label != "" && version != "" { + return label, version, nil + } + } + } + return "", "", fmt.Errorf("cannot infer release via /etc/lsb-release, see the --release option") +} + +// obtainRelease returns the Chisel release information matching the provided string, +// fetching it if necessary. The provided string should be either: +// * "-", +// * the path to a directory containing a previously fetched release, +// * "" and Chisel will attempt to read the release label from the host. +func obtainRelease(releaseStr string) (release *setup.Release, err error) { + if strings.Contains(releaseStr, "/") { + release, err = setup.ReadRelease(releaseStr) + } else { + var label, version string + if releaseStr == "" { + label, version, err = readReleaseInfo() + } else { + label, version, err = parseReleaseInfo(releaseStr) + } + if err != nil { + return nil, err + } + release, err = setup.FetchRelease(&setup.FetchOptions{ + Label: label, + Version: version, + }) + } + if err != nil { + return nil, err + } + return release, nil +} diff --git a/go.mod b/go.mod index 3dc67f47..d59245bd 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/canonical/chisel -go 1.20 +go 1.21 require ( github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb diff --git a/internal/deb/extract.go b/internal/deb/extract.go index eb15ed7e..07f219bd 100644 --- a/internal/deb/extract.go +++ b/internal/deb/extract.go @@ -26,30 +26,33 @@ type ExtractOptions struct { TargetDir string Extract map[string][]ExtractInfo // Create can optionally be set to control the creation of extracted entries. - // extractInfo is set to the matching entry in Extract, and is nil in cases where + // extractInfos is set to the matching entries in Extract, and is nil in cases where // the created entry is implicit and unlisted (for example, parent directories). - Create func(extractInfo *ExtractInfo, options *fsutil.CreateOptions) error + Create func(extractInfos []ExtractInfo, options *fsutil.CreateOptions) error } type ExtractInfo struct { Path string Mode uint Optional bool + Context any } func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) { for extractPath, extractInfos := range options.Extract { isGlob := strings.ContainsAny(extractPath, "*?") if isGlob { - if len(extractInfos) != 1 || extractInfos[0].Path != extractPath || extractInfos[0].Mode != 0 { - return nil, fmt.Errorf("when using wildcards source and target paths must match: %s", extractPath) + for _, extractInfo := range extractInfos { + if extractInfo.Path != extractPath || extractInfo.Mode != 0 { + return nil, fmt.Errorf("when using wildcards source and target paths must match: %s", extractPath) + } } } } if options.Create == nil { validOpts := *options - validOpts.Create = func(_ *ExtractInfo, o *fsutil.CreateOptions) error { + validOpts.Create = func(_ []ExtractInfo, o *fsutil.CreateOptions) error { _, err := fsutil.Create(o) return err } @@ -123,33 +126,6 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { syscall.Umask(oldUmask) }() - shouldExtract := func(pkgPath string) (globPath string, ok bool) { - if pkgPath == "" { - return "", false - } - pkgPathIsDir := pkgPath[len(pkgPath)-1] == '/' - for extractPath, extractInfos := range options.Extract { - if extractPath == "" { - continue - } - switch { - case strings.ContainsAny(extractPath, "*?"): - if strdist.GlobPath(extractPath, pkgPath) { - return extractPath, true - } - case extractPath == pkgPath: - return "", true - case pkgPathIsDir: - for _, extractInfo := range extractInfos { - if strings.HasPrefix(extractInfo.Path, pkgPath) { - return "", true - } - } - } - } - return "", false - } - pendingPaths := make(map[string]bool) for extractPath, extractInfos := range options.Extract { for _, extractInfo := range extractInfos { @@ -182,8 +158,7 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { continue } sourcePath = sourcePath[1:] - globPath, ok := shouldExtract(sourcePath) - if !ok { + if sourcePath == "" { continue } @@ -192,22 +167,32 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { tarDirMode[sourcePath] = tarHeader.FileInfo().Mode() } - //debugf("Extracting header: %#v", tarHeader) - - var extractInfos []ExtractInfo - if globPath != "" { - extractInfos = options.Extract[globPath] - delete(pendingPaths, globPath) - } else { - extractInfos, ok = options.Extract[sourcePath] - if !ok { + // Find all globs and copies that require this source, and map them by + // their target paths on disk. + targetPaths := map[string][]ExtractInfo{} + for extractPath, extractInfos := range options.Extract { + if extractPath == "" { continue } - delete(pendingPaths, sourcePath) + if strings.ContainsAny(extractPath, "*?") { + if strdist.GlobPath(extractPath, sourcePath) { + targetPaths[sourcePath] = append(targetPaths[sourcePath], extractInfos...) + delete(pendingPaths, extractPath) + } + } else if extractPath == sourcePath { + for _, extractInfo := range extractInfos { + targetPaths[extractInfo.Path] = append(targetPaths[extractInfo.Path], extractInfo) + } + delete(pendingPaths, extractPath) + } + } + if len(targetPaths) == 0 { + // Nothing to do. + continue } var contentCache []byte - var contentIsCached = len(extractInfos) > 1 && !sourceIsDir && globPath == "" + var contentIsCached = len(targetPaths) > 1 && !sourceIsDir if contentIsCached { // Read and cache the content so it may be reused. // As an alternative, to avoid having an entire file in @@ -222,21 +207,24 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { } var pathReader io.Reader = tarReader - for _, extractInfo := range extractInfos { + for targetPath, extractInfos := range targetPaths { if contentIsCached { pathReader = bytes.NewReader(contentCache) } - var relPath string - if globPath == "" { - relPath = extractInfo.Path - } else { - relPath = sourcePath + mode := extractInfos[0].Mode + for _, extractInfo := range extractInfos { + if extractInfo.Mode != mode { + if mode < extractInfo.Mode { + mode, extractInfo.Mode = extractInfo.Mode, mode + } + return fmt.Errorf("path %s requested twice with diverging mode: 0%03o != 0%03o", targetPath, mode, extractInfo.Mode) + } } - if extractInfo.Mode != 0 { - tarHeader.Mode = int64(extractInfo.Mode) + if mode != 0 { + tarHeader.Mode = int64(mode) } // Create the parent directories using the permissions from the tarball. - parents := parentDirs(relPath) + parents := parentDirs(targetPath) for _, path := range parents { if path == "/" { continue @@ -259,19 +247,16 @@ func extractData(dataReader io.Reader, options *ExtractOptions) error { } // Create the entry itself. createOptions := &fsutil.CreateOptions{ - Path: filepath.Join(options.TargetDir, relPath), + Path: filepath.Join(options.TargetDir, targetPath), Mode: tarHeader.FileInfo().Mode(), Data: pathReader, Link: tarHeader.Linkname, MakeParents: true, } - err := options.Create(&extractInfo, createOptions) + err := options.Create(extractInfos, createOptions) if err != nil { return err } - if globPath != "" { - break - } } } diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index f3ea26b2..22a1fd18 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -157,7 +157,7 @@ var extractTests = []extractTest{{ "/dir/": "dir 0755", "/dir/several/": "dir 0755", }, - notCreated: []string{"/dir/"}, + notCreated: []string{}, }, { summary: "Globbing for files with multiple levels at once", pkgdata: testutil.PackageData["test-package"], @@ -175,7 +175,7 @@ var extractTests = []extractTest{{ "/dir/several/levels/deep/": "dir 0755", "/dir/several/levels/deep/file": "file 0644 6bc26dff", }, - notCreated: []string{"/dir/"}, + notCreated: []string{}, }, { summary: "Globbing multiple paths", pkgdata: testutil.PackageData["test-package"], @@ -197,7 +197,7 @@ var extractTests = []extractTest{{ "/dir/several/levels/deep/": "dir 0755", "/dir/several/levels/deep/file": "file 0644 6bc26dff", }, - notCreated: []string{"/dir/"}, + notCreated: []string{}, }, { summary: "Globbing must have matching source and target", pkgdata: testutil.PackageData["test-package"], @@ -208,7 +208,7 @@ var extractTests = []extractTest{{ }}, }, }, - error: `cannot extract .*: when using wildcards source and target paths must match: /foo/b\*\*`, + error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /foo/b\*\*`, }, { summary: "Globbing must also have a single target", pkgdata: testutil.PackageData["test-package"], @@ -221,7 +221,7 @@ var extractTests = []extractTest{{ }}, }, }, - error: `cannot extract .*: when using wildcards source and target paths must match: /foo/b\*\*`, + error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /foo/b\*\*`, }, { summary: "Globbing cannot change modes", pkgdata: testutil.PackageData["test-package"], @@ -233,7 +233,7 @@ var extractTests = []extractTest{{ }}, }, }, - error: `cannot extract .*: when using wildcards source and target paths must match: /dir/n\*\*`, + error: `cannot extract from package "test-package": when using wildcards source and target paths must match: /dir/n\*\*`, }, { summary: "Missing file", pkgdata: testutil.PackageData["test-package"], @@ -337,6 +337,21 @@ var extractTests = []extractTest{{ "/日本/語": "file 0644 85738f8f", }, notCreated: []string{}, +}, { + summary: "Entries for same destination must have the same mode", + pkgdata: testutil.PackageData["test-package"], + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/dir/": []deb.ExtractInfo{{ + Path: "/dir/", + Mode: 0777, + }}, + "/d**": []deb.ExtractInfo{{ + Path: "/d**", + }}, + }, + }, + error: `cannot extract from package "test-package": path /dir/ requested twice with diverging mode: 0777 != 0000`, }} func (s *S) TestExtract(c *C) { @@ -348,7 +363,7 @@ func (s *S) TestExtract(c *C) { options.Package = "test-package" options.TargetDir = dir createdPaths := make(map[string]bool) - options.Create = func(_ *deb.ExtractInfo, o *fsutil.CreateOptions) error { + options.Create = func(_ []deb.ExtractInfo, o *fsutil.CreateOptions) error { relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) if o.Mode.IsDir() { relPath = relPath + "/" @@ -386,3 +401,96 @@ func (s *S) TestExtract(c *C) { c.Assert(result, DeepEquals, test.result) } } + +var extractCreateCallbackTests = []struct { + summary string + pkgdata []byte + options deb.ExtractOptions + calls map[string][]deb.ExtractInfo +}{{ + summary: "Create is called with the set of ExtractInfo(s) that match the file", + pkgdata: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0755, "./"), + testutil.Dir(0766, "./dir/"), + testutil.Reg(0644, "./dir/file", "whatever"), + }), + options: deb.ExtractOptions{ + Extract: map[string][]deb.ExtractInfo{ + "/dir/": []deb.ExtractInfo{{ + Path: "/dir/", + }}, + "/d**": []deb.ExtractInfo{{ + Path: "/d**", + }}, + "/d?r/": []deb.ExtractInfo{{ + Path: "/d?r/", + }}, + "/dir/file": []deb.ExtractInfo{{ + Path: "/dir/file", + }, { + Path: "/dir/file-cpy", + }}, + "/foo/": []deb.ExtractInfo{{ + Path: "/foo/", + Optional: true, + }}, + }, + }, + calls: map[string][]deb.ExtractInfo{ + "/dir/": []deb.ExtractInfo{ + deb.ExtractInfo{ + Path: "/d**", + }, + deb.ExtractInfo{ + Path: "/d?r/", + }, + deb.ExtractInfo{ + Path: "/dir/", + }, + }, + "/dir/file": []deb.ExtractInfo{ + deb.ExtractInfo{ + Path: "/d**", + }, + deb.ExtractInfo{ + Path: "/dir/file", + }, + }, + "/dir/file-cpy": []deb.ExtractInfo{ + deb.ExtractInfo{ + Path: "/dir/file-cpy", + }, + }, + }, +}} + +func (s *S) TestExtractCreateCallback(c *C) { + for _, test := range extractCreateCallbackTests { + c.Logf("Test: %s", test.summary) + dir := c.MkDir() + options := test.options + options.Package = "test-package" + options.TargetDir = dir + createExtractInfos := map[string][]deb.ExtractInfo{} + options.Create = func(extractInfos []deb.ExtractInfo, o *fsutil.CreateOptions) error { + if extractInfos == nil { + // Creating implicit parent directories, we don't care about those. + return nil + } + relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir)) + if o.Mode.IsDir() { + relPath = relPath + "/" + } + sort.Slice(extractInfos, func(i, j int) bool { + return extractInfos[i].Path < extractInfos[j].Path + }) + createExtractInfos[relPath] = extractInfos + return nil + } + + err := deb.Extract(bytes.NewBuffer(test.pkgdata), &options) + c.Assert(err, IsNil) + + c.Assert(createExtractInfos, DeepEquals, test.calls) + } +} diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 90bcfb5e..48b12cb7 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -45,6 +45,7 @@ func Create(options *CreateOptions) (*Entry, error) { return nil, err } } + switch o.Mode & fs.ModeType { case 0: err = createFile(o) @@ -60,9 +61,13 @@ func Create(options *CreateOptions) (*Entry, error) { return nil, err } + s, err := os.Lstat(o.Path) + if err != nil { + return nil, err + } entry := &Entry{ Path: o.Path, - Mode: o.Mode, + Mode: s.Mode(), Hash: hash, Size: rp.size, Link: o.Link, diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index 50c0c746..7288d878 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -2,7 +2,6 @@ package fsutil_test import ( "bytes" - "fmt" "io/fs" "os" "path/filepath" @@ -80,6 +79,20 @@ var createTests = []createTest{{ // mode is not updated. "/foo/": "dir 0765", }, +}, { + options: fsutil.CreateOptions{ + Path: "foo", + // Mode should be ignored for existing entry. + Mode: 0644, + Data: bytes.NewBufferString("changed"), + }, + hackdir: func(c *C, dir string) { + c.Assert(os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0666), IsNil) + }, + result: map[string]string{ + // mode is not updated. + "/foo": "file 0666 d67e2e94", + }, }} func (s *S) TestCreate(c *C) { @@ -111,33 +124,12 @@ func (s *S) TestCreate(c *C) { c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) // [fsutil.Create] does not return information about parent directories // created implicitly. We only check for the requested path. - c.Assert(dumpFSEntry(entry, dir)[test.options.Path], DeepEquals, test.result[test.options.Path]) - } -} - -// dumpFSEntry returns the file entry in the same format as [testutil.TreeDump]. -func dumpFSEntry(fsEntry *fsutil.Entry, root string) map[string]string { - result := make(map[string]string) - path := strings.TrimPrefix(fsEntry.Path, root) - fperm := fsEntry.Mode.Perm() - if fsEntry.Mode&fs.ModeSticky != 0 { - fperm |= 01000 - } - switch fsEntry.Mode.Type() { - case fs.ModeDir: - result[path+"/"] = fmt.Sprintf("dir %#o", fperm) - case fs.ModeSymlink: - result[path] = fmt.Sprintf("symlink %s", fsEntry.Link) - case 0: // Regular - var entry string - if fsEntry.Size == 0 { - entry = fmt.Sprintf("file %#o empty", fsEntry.Mode.Perm()) - } else { - entry = fmt.Sprintf("file %#o %s", fperm, fsEntry.Hash[:8]) + entry.Path = strings.TrimPrefix(entry.Path, dir) + // Add the slashes that TreeDump adds to the path. + slashPath := "/" + test.options.Path + if test.options.Mode.IsDir() { + slashPath = slashPath + "/" } - result[path] = entry - default: - panic(fmt.Errorf("unknown file type %d: %s", fsEntry.Mode.Type(), path)) + c.Assert(testutil.TreeDumpEntry(entry), DeepEquals, test.result[slashPath]) } - return result } diff --git a/internal/scripts/scripts.go b/internal/scripts/scripts.go index f4c83352..eead1a81 100644 --- a/internal/scripts/scripts.go +++ b/internal/scripts/scripts.go @@ -1,13 +1,16 @@ package scripts import ( - "go.starlark.net/resolve" - "go.starlark.net/starlark" - + "bytes" "fmt" "os" "path/filepath" "strings" + + "go.starlark.net/resolve" + "go.starlark.net/starlark" + + "github.com/canonical/chisel/internal/fsutil" ) func init() { @@ -33,6 +36,9 @@ type ContentValue struct { RootDir string CheckRead func(path string) error CheckWrite func(path string) error + // OnWrite has to be called after a successful write with the entry resulting + // from the write. + OnWrite func(entry *fsutil.Entry) error } // Content starlark.Value interface @@ -171,10 +177,18 @@ func (c *ContentValue) Write(thread *starlark.Thread, fn *starlark.Builtin, args // No mode parameter for now as slices are supposed to list files // explicitly instead. - err = os.WriteFile(fpath, fdata, 0644) + entry, err := fsutil.Create(&fsutil.CreateOptions{ + Path: fpath, + Data: bytes.NewReader(fdata), + Mode: 0644, + }) if err != nil { return nil, c.polishError(path, err) } + err = c.OnWrite(entry) + if err != nil { + return nil, err + } return starlark.None, nil } diff --git a/internal/scripts/scripts_test.go b/internal/scripts/scripts_test.go index 211e5545..f86711a6 100644 --- a/internal/scripts/scripts_test.go +++ b/internal/scripts/scripts_test.go @@ -4,9 +4,11 @@ import ( "fmt" "os" "path/filepath" + "strings" . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/testutil" ) @@ -17,6 +19,7 @@ type scriptsTest struct { hackdir func(c *C, dir string) script string result map[string]string + mutated map[string]string checkr func(path string) error checkw func(path string) error error string @@ -44,6 +47,10 @@ var scriptsTests = []scriptsTest{{ "/foo/file1.txt": "file 0644 5b41362b", "/foo/file2.txt": "file 0644 d98cf53e", }, + mutated: map[string]string{ + "/foo/file1.txt": "file 0644 5b41362b", + "/foo/file2.txt": "file 0644 d98cf53e", + }, }, { summary: "Read a file", content: map[string]string{ @@ -59,6 +66,9 @@ var scriptsTests = []scriptsTest{{ "/foo/file1.txt": "file 0644 5b41362b", "/foo/file2.txt": "file 0644 5b41362b", }, + mutated: map[string]string{ + "/foo/file2.txt": "file 0644 5b41362b", + }, }, { summary: "List a directory", content: map[string]string{ @@ -77,6 +87,53 @@ var scriptsTests = []scriptsTest{{ "/bar/": "dir 0755", "/bar/file3.txt": "file 0644 5b41362b", }, +}, { + summary: "OnWrite is called for modified files only", + content: map[string]string{ + "foo/file1.txt": `placeholder`, + "foo/file2.txt": `placeholder`, + // This file is not mutable, it cannot be written to. + "foo/file3.txt": `placeholder`, + }, + script: ` + content.write("/foo/file1.txt", "data1") + content.write("/foo/file2.txt", "data2") + `, + checkw: func(p string) error { + if p == "foo/file3.txt" { + return fmt.Errorf("no write: %s", p) + } + return nil + }, + result: map[string]string{ + "/foo/": "dir 0755", + "/foo/file1.txt": "file 0644 5b41362b", + "/foo/file2.txt": "file 0644 d98cf53e", + "/foo/file3.txt": "file 0644 40978892", + }, + mutated: map[string]string{ + "/foo/file1.txt": "file 0644 5b41362b", + "/foo/file2.txt": "file 0644 d98cf53e", + }, +}, { + summary: "Mode is not changed when writing to a file", + content: map[string]string{ + "foo/file1.txt": ``, + "foo/file2.txt": ``, + }, + hackdir: func(c *C, dir string) { + fpath1 := filepath.Join(dir, "foo/file1.txt") + _ = os.Chmod(fpath1, 0744) + }, + script: ` + content.write("/foo/file1.txt", "data1") + content.write("/foo/file2.txt", "data2") + `, + result: map[string]string{ + "/foo/": "dir 0755", + "/foo/file1.txt": "file 0744 5b41362b", + "/foo/file2.txt": "file 0644 d98cf53e", + }, }, { summary: "Forbid relative paths", content: map[string]string{ @@ -217,10 +274,17 @@ func (s *S) TestScripts(c *C) { test.hackdir(c, rootDir) } + mutatedFiles := map[string]string{} content := &scripts.ContentValue{ RootDir: rootDir, CheckRead: test.checkr, CheckWrite: test.checkw, + OnWrite: func(entry *fsutil.Entry) error { + // Set relative path. + entry.Path = strings.TrimPrefix(entry.Path, rootDir) + mutatedFiles[entry.Path] = testutil.TreeDumpEntry(entry) + return nil + }, } namespace := map[string]scripts.Value{ "content": content, @@ -237,6 +301,10 @@ func (s *S) TestScripts(c *C) { } c.Assert(testutil.TreeDump(rootDir), DeepEquals, test.result) + + if test.mutated != nil { + c.Assert(mutatedFiles, DeepEquals, test.mutated) + } } } diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 914c9606..ef604fcd 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "regexp" + "slices" "strings" "golang.org/x/crypto/openpgp/packet" @@ -32,8 +33,8 @@ type Archive struct { Version string Suites []string Components []string + Priority int Pro string - Priority int32 PubKeys []*packet.PublicKey } @@ -190,6 +191,28 @@ func (r *Release) validate() error { paths[newPath] = new } + // Check for archive priority conflicts. + priorities := make(map[int]*Archive) + for _, archive := range r.Archives { + if old, ok := priorities[archive.Priority]; ok { + if old.Name > archive.Name { + archive, old = old, archive + } + return fmt.Errorf("chisel.yaml: archives %q and %q have the same priority value of %v", old.Name, archive.Name, archive.Priority) + } + priorities[archive.Priority] = archive + } + + // Check that archives pinned in packages are defined. + for _, pkg := range r.Packages { + if pkg.Archive == "" { + continue + } + if _, ok := r.Archives[pkg.Archive]; !ok { + return fmt.Errorf("%s: package refers to undefined archive %q", pkg.Path, pkg.Archive) + } + } + return nil } @@ -340,12 +363,17 @@ const ( proInfra proValue = "infra" ) +const ( + MaxArchivePriority = 1000 + MinArchivePriority = -1000 +) + type yamlArchive struct { Version string `yaml:"version"` Suites []string `yaml:"suites"` Components []string `yaml:"components"` Default bool `yaml:"default"` - Priority int32 `yaml:"priority"` + Priority int `yaml:"priority"` Pro proValue `yaml:"pro"` PubKeys []string `yaml:"public-keys"` // V1PubKeys is used for compatibility with format "chisel-v1". @@ -508,6 +536,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { } archiveKeys = append(archiveKeys, key) } + if details.Priority > MaxArchivePriority || details.Priority < MinArchivePriority { + return nil, fmt.Errorf("%s: archive %q has invalid priority value %d", fileName, archiveName, details.Priority) + } release.Archives[archiveName] = &Archive{ Name: archiveName, Version: details.Version, @@ -564,11 +595,8 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro // Do not add the slice to its own essentials list. continue } - // TODO replace with slices.Contains once it is stable. - for _, sk := range slice.Essential { - if sk == sliceKey { - return nil, fmt.Errorf("package %s defined with redundant essential slice: %s", pkgName, refName) - } + if slices.Contains(slice.Essential, sliceKey) { + return nil, fmt.Errorf("package %s defined with redundant essential slice: %s", pkgName, refName) } slice.Essential = append(slice.Essential, sliceKey) } @@ -580,11 +608,8 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro if sliceKey.Package == slice.Package && sliceKey.Slice == slice.Name { return nil, fmt.Errorf("cannot add slice to itself as essential %q in %s", refName, pkgPath) } - // TODO replace with slices.Contains once it is stable. - for _, sk := range slice.Essential { - if sk == sliceKey { - return nil, fmt.Errorf("slice %s defined with redundant essential slice: %s", slice, refName) - } + if slices.Contains(slice.Essential, sliceKey) { + return nil, fmt.Errorf("slice %s defined with redundant essential slice: %s", slice, refName) } slice.Essential = append(slice.Essential, sliceKey) } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index ad79f385..9da5a9f9 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -758,7 +758,7 @@ var setupTests = []setupTest{{ }, }, }, { - summary: "Multiple archives", + summary: "Multiple archives with priorities", input: map[string]string{ "chisel.yaml": ` format: chisel-v1 @@ -774,7 +774,7 @@ var setupTests = []setupTest{{ version: 22.04 components: [universe] suites: [jammy-updates] - priority: 10 + priority: -10 v1-public-keys: [test-key] v1-public-keys: test-key: @@ -803,7 +803,7 @@ var setupTests = []setupTest{{ Version: "22.04", Suites: []string{"jammy-updates"}, Components: []string{"universe"}, - Priority: 10, + Priority: -10, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, }, @@ -816,6 +816,53 @@ var setupTests = []setupTest{{ }, }, }, +}, { + summary: "Two archives cannot have same priority", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + suites: [jammy] + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [universe] + suites: [jammy-updates] + priority: 20 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + relerror: `chisel.yaml: archives "bar" and "foo" have the same priority value of 20`, +}, { + summary: "Invalid archive priority", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + suites: [jammy] + priority: 10000 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + }, + relerror: `chisel.yaml: archive "foo" has invalid priority value 10000`, }, { summary: "Extra fields in YAML are ignored (necessary for forward compatibility)", input: map[string]string{ @@ -884,12 +931,13 @@ var setupTests = []setupTest{{ components: [main, universe] suites: [jammy] v1-public-keys: [extra-key] - default: true + priority: 20 bar: version: 22.04 components: [universe] suites: [jammy-updates] v1-public-keys: [test-key, extra-key] + priority: 10 v1-public-keys: extra-key: id: ` + extraTestKey.ID + ` @@ -903,14 +951,13 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "foo", - Archives: map[string]*setup.Archive{ "foo": { Name: "foo", Version: "22.04", Suites: []string{"jammy"}, Components: []string{"main", "universe"}, + Priority: 20, PubKeys: []*packet.PublicKey{extraTestKey.PubKey}, }, "bar": { @@ -918,6 +965,7 @@ var setupTests = []setupTest{{ Version: "22.04", Suites: []string{"jammy-updates"}, Components: []string{"universe"}, + Priority: 10, PubKeys: []*packet.PublicKey{testKey.PubKey, extraTestKey.PubKey}, }, }, @@ -1287,6 +1335,30 @@ var setupTests = []setupTest{{ `, }, relerror: `package "mypkg" has invalid essential slice reference: "mypkg-slice"`, +}, { + summary: "Glob clashes within same package", + input: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/**: + myslice2: + contents: + /dir/file: {text: "foo"} + `, + }, + // TODO this should be an error because the content does not match. +}, { + summary: "Pinned archive is not defined", + input: map[string]string{ + "slices/test-package.yaml": ` + package: test-package + archive: non-existing + `, + }, + relerror: `slices/test-package.yaml: package refers to undefined archive "non-existing"`, }, { summary: "Pro values in archives", input: map[string]string{ diff --git a/internal/slicer/report.go b/internal/slicer/report.go index 22600399..8897f518 100644 --- a/internal/slicer/report.go +++ b/internal/slicer/report.go @@ -11,12 +11,13 @@ import ( ) type ReportEntry struct { - Path string - Mode fs.FileMode - Hash string - Size int - Slices map[*setup.Slice]bool - Link string + Path string + Mode fs.FileMode + Hash string + Size int + Slices map[*setup.Slice]bool + Link string + FinalHash string } // Report holds the information about files and directories created when slicing @@ -30,31 +31,32 @@ type Report struct { // NewReport returns an empty report for content that will be based at the // provided root path. -func NewReport(root string) *Report { - return &Report{ +func NewReport(root string) (*Report, error) { + if !filepath.IsAbs(root) { + return nil, fmt.Errorf("cannot use relative path for report root: %q", root) + } + report := &Report{ Root: filepath.Clean(root) + "/", Entries: make(map[string]ReportEntry), } + return report, nil } func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error { - if !strings.HasPrefix(fsEntry.Path, r.Root) { - return fmt.Errorf("cannot add path %q outside of root %q", fsEntry.Path, r.Root) - } - relPath := filepath.Clean("/" + strings.TrimPrefix(fsEntry.Path, r.Root)) - if fsEntry.Mode.IsDir() { - relPath = relPath + "/" + relPath, err := r.sanitizeAbsPath(fsEntry.Path, fsEntry.Mode.IsDir()) + if err != nil { + return fmt.Errorf("cannot add path to report: %s", err) } if entry, ok := r.Entries[relPath]; ok { if fsEntry.Mode != entry.Mode { - return fmt.Errorf("path %q reported twice with diverging mode: %q != %q", relPath, fsEntry.Mode, entry.Mode) + return fmt.Errorf("path %s reported twice with diverging mode: 0%03o != 0%03o", relPath, fsEntry.Mode, entry.Mode) } else if fsEntry.Link != entry.Link { - return fmt.Errorf("path %q reported twice with diverging link: %q != %q", relPath, fsEntry.Link, entry.Link) + return fmt.Errorf("path %s reported twice with diverging link: %q != %q", relPath, fsEntry.Link, entry.Link) } else if fsEntry.Size != entry.Size { - return fmt.Errorf("path %q reported twice with diverging size: %d != %d", relPath, fsEntry.Size, entry.Size) + return fmt.Errorf("path %s reported twice with diverging size: %d != %d", relPath, fsEntry.Size, entry.Size) } else if fsEntry.Hash != entry.Hash { - return fmt.Errorf("path %q reported twice with diverging hash: %q != %q", relPath, fsEntry.Hash, entry.Hash) + return fmt.Errorf("path %s reported twice with diverging hash: %q != %q", relPath, fsEntry.Hash, entry.Hash) } entry.Slices[slice] = true r.Entries[relPath] = entry @@ -70,3 +72,38 @@ func (r *Report) Add(slice *setup.Slice, fsEntry *fsutil.Entry) error { } return nil } + +// Mutate updates the FinalHash and Size of an existing path entry. +func (r *Report) Mutate(fsEntry *fsutil.Entry) error { + relPath, err := r.sanitizeAbsPath(fsEntry.Path, fsEntry.Mode.IsDir()) + if err != nil { + return fmt.Errorf("cannot mutate path in report: %s", err) + } + + entry, ok := r.Entries[relPath] + if !ok { + return fmt.Errorf("cannot mutate path in report: %s not previously added", relPath) + } + if entry.Mode.IsDir() { + return fmt.Errorf("cannot mutate path in report: %s is a directory", relPath) + } + if entry.Hash == fsEntry.Hash { + // Content has not changed, nothing to do. + return nil + } + entry.FinalHash = fsEntry.Hash + entry.Size = fsEntry.Size + r.Entries[relPath] = entry + return nil +} + +func (r *Report) sanitizeAbsPath(path string, isDir bool) (relPath string, err error) { + if !strings.HasPrefix(path, r.Root) { + return "", fmt.Errorf("%s outside of root %s", path, r.Root) + } + relPath = filepath.Clean("/" + strings.TrimPrefix(path, r.Root)) + if isDir { + relPath = relPath + "/" + } + return relPath, nil +} diff --git a/internal/slicer/report_test.go b/internal/slicer/report_test.go index 0b04f70e..762b35ad 100644 --- a/internal/slicer/report_test.go +++ b/internal/slicer/report_test.go @@ -27,25 +27,31 @@ var otherSlice = &setup.Slice{ } var sampleDir = fsutil.Entry{ - Path: "/base/exampleDir/", + Path: "/base/example-dir/", Mode: fs.ModeDir | 0654, Link: "", } var sampleFile = fsutil.Entry{ - Path: "/base/exampleFile", + Path: "/base/example-file", Mode: 0777, - Hash: "exampleFile_hash", + Hash: "example-file_hash", Size: 5678, Link: "", } var sampleLink = fsutil.Entry{ - Path: "/base/exampleLink", + Path: "/base/example-link", Mode: 0777, - Hash: "exampleFile_hash", + Hash: "example-file_hash", Size: 5678, - Link: "/base/exampleFile", + Link: "/base/example-file", +} + +var sampleFileMutated = fsutil.Entry{ + Path: sampleFile.Path, + Hash: sampleFile.Hash + "_changed", + Size: sampleFile.Size + 10, } type sliceAndEntry struct { @@ -56,6 +62,7 @@ type sliceAndEntry struct { var reportTests = []struct { summary string add []sliceAndEntry + mutate []*fsutil.Entry // indexed by path. expected map[string]slicer.ReportEntry // error after adding the last [sliceAndEntry]. @@ -64,8 +71,8 @@ var reportTests = []struct { summary: "Regular directory", add: []sliceAndEntry{{entry: sampleDir, slice: oneSlice}}, expected: map[string]slicer.ReportEntry{ - "/exampleDir/": { - Path: "/exampleDir/", + "/example-dir/": { + Path: "/example-dir/", Mode: fs.ModeDir | 0654, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", @@ -77,8 +84,8 @@ var reportTests = []struct { {entry: sampleDir, slice: otherSlice}, }, expected: map[string]slicer.ReportEntry{ - "/exampleDir/": { - Path: "/exampleDir/", + "/example-dir/": { + Path: "/example-dir/", Mode: fs.ModeDir | 0654, Slices: map[*setup.Slice]bool{oneSlice: true, otherSlice: true}, Link: "", @@ -87,10 +94,10 @@ var reportTests = []struct { summary: "Regular file", add: []sliceAndEntry{{entry: sampleFile, slice: oneSlice}}, expected: map[string]slicer.ReportEntry{ - "/exampleFile": { - Path: "/exampleFile", + "/example-file": { + Path: "/example-file", Mode: 0777, - Hash: "exampleFile_hash", + Hash: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", @@ -99,13 +106,13 @@ var reportTests = []struct { summary: "Regular file link", add: []sliceAndEntry{{entry: sampleLink, slice: oneSlice}}, expected: map[string]slicer.ReportEntry{ - "/exampleLink": { - Path: "/exampleLink", + "/example-link": { + Path: "/example-link", Mode: 0777, - Hash: "exampleFile_hash", + Hash: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, - Link: "/base/exampleFile", + Link: "/base/example-file", }}, }, { summary: "Several entries", @@ -114,16 +121,16 @@ var reportTests = []struct { {entry: sampleFile, slice: otherSlice}, }, expected: map[string]slicer.ReportEntry{ - "/exampleDir/": { - Path: "/exampleDir/", + "/example-dir/": { + Path: "/example-dir/", Mode: fs.ModeDir | 0654, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", }, - "/exampleFile": { - Path: "/exampleFile", + "/example-file": { + Path: "/example-file", Mode: 0777, - Hash: "exampleFile_hash", + Hash: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{otherSlice: true}, Link: "", @@ -135,10 +142,10 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, }, expected: map[string]slicer.ReportEntry{ - "/exampleFile": { - Path: "/exampleFile", + "/example-file": { + Path: "/example-file", Mode: 0777, - Hash: "exampleFile_hash", + Hash: "example-file_hash", Size: 5678, Slices: map[*setup.Slice]bool{oneSlice: true}, Link: "", @@ -155,7 +162,7 @@ var reportTests = []struct { Link: sampleFile.Link, }, slice: oneSlice}, }, - err: `path "/exampleFile" reported twice with diverging mode: "----------" != "-rwxrwxrwx"`, + err: `path /example-file reported twice with diverging mode: 0000 != 0777`, }, { summary: "Error for same path distinct hash", add: []sliceAndEntry{ @@ -168,7 +175,7 @@ var reportTests = []struct { Link: sampleFile.Link, }, slice: oneSlice}, }, - err: `path "/exampleFile" reported twice with diverging hash: "distinct hash" != "exampleFile_hash"`, + err: `path /example-file reported twice with diverging hash: "distinct hash" != "example-file_hash"`, }, { summary: "Error for same path distinct size", add: []sliceAndEntry{ @@ -181,7 +188,7 @@ var reportTests = []struct { Link: sampleFile.Link, }, slice: oneSlice}, }, - err: `path "/exampleFile" reported twice with diverging size: 0 != 5678`, + err: `path /example-file reported twice with diverging size: 0 != 5678`, }, { summary: "Error for same path distinct link", add: []sliceAndEntry{ @@ -194,28 +201,85 @@ var reportTests = []struct { Link: "distinct link", }, slice: oneSlice}, }, - err: `path "/exampleFile" reported twice with diverging link: "distinct link" != ""`, + err: `path /example-file reported twice with diverging link: "distinct link" != ""`, }, { summary: "Error for path outside root", add: []sliceAndEntry{ {entry: fsutil.Entry{Path: "/file"}, slice: oneSlice}, }, - err: `cannot add path "/file" outside of root "/base/"`, + err: `cannot add path to report: /file outside of root /base/`, +}, { + summary: "Error for mutated path outside root", + mutate: []*fsutil.Entry{{Path: "/file"}}, + err: `cannot mutate path in report: /file outside of root /base/`, }, { summary: "File name has root prefix but without the directory slash", add: []sliceAndEntry{ {entry: fsutil.Entry{Path: "/basefile"}, slice: oneSlice}, }, - err: `cannot add path "/basefile" outside of root "/base/"`, + err: `cannot add path to report: /basefile outside of root /base/`, +}, { + summary: "Add mutated regular file", + add: []sliceAndEntry{ + {entry: sampleFile, slice: oneSlice}, + {entry: sampleDir, slice: oneSlice}, + }, + mutate: []*fsutil.Entry{&sampleFileMutated}, + expected: map[string]slicer.ReportEntry{ + "/example-dir/": { + Path: "/example-dir/", + Mode: fs.ModeDir | 0654, + Slices: map[*setup.Slice]bool{oneSlice: true}, + Link: "", + }, + "/example-file": { + Path: "/example-file", + Mode: 0777, + Hash: "example-file_hash", + Size: 5688, + Slices: map[*setup.Slice]bool{oneSlice: true}, + Link: "", + FinalHash: "example-file_hash_changed", + }}, +}, { + summary: "Calling mutated with identical content to initial file", + add: []sliceAndEntry{ + {entry: sampleFile, slice: oneSlice}, + }, + mutate: []*fsutil.Entry{&sampleFile}, + expected: map[string]slicer.ReportEntry{ + "/example-file": { + Path: "/example-file", + Mode: 0777, + Hash: "example-file_hash", + Size: 5678, + Slices: map[*setup.Slice]bool{oneSlice: true}, + Link: "", + // FinalHash is not updated. + FinalHash: "", + }}, +}, { + summary: "Mutated paths must refer to previously added entries", + mutate: []*fsutil.Entry{&sampleFileMutated}, + err: `cannot mutate path in report: /example-file not previously added`, +}, { + summary: "Cannot mutate directory", + add: []sliceAndEntry{{entry: sampleDir, slice: oneSlice}}, + mutate: []*fsutil.Entry{&sampleDir}, + err: `cannot mutate path in report: /example-dir/ is a directory`, }} -func (s *S) TestReportAdd(c *C) { +func (s *S) TestReport(c *C) { for _, test := range reportTests { - report := slicer.NewReport("/base/") var err error + report, err := slicer.NewReport("/base/") + c.Assert(err, IsNil) for _, si := range test.add { err = report.Add(si.slice, &si.entry) } + for _, e := range test.mutate { + err = report.Mutate(e) + } if test.err != "" { c.Assert(err, ErrorMatches, test.err) continue @@ -224,3 +288,8 @@ func (s *S) TestReportAdd(c *C) { c.Assert(report.Entries, DeepEquals, test.expected, Commentf(test.summary)) } } + +func (s *S) TestRootRelativePath(c *C) { + _, err := slicer.NewReport("../base/") + c.Assert(err, ErrorMatches, `cannot use relative path for report root: "../base/"`) +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 39697210..4b757ec9 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -7,6 +7,8 @@ import ( "io" "os" "path/filepath" + "slices" + "sort" "strings" "syscall" @@ -18,93 +20,106 @@ import ( ) type RunOptions struct { - Selection *setup.Selection - PkgArchives map[string]archive.Archive - TargetDir string + Selection *setup.Selection + Archives map[string]archive.Archive + TargetDir string } -func Run(options *RunOptions) (*Report, error) { +type pathData struct { + until setup.PathUntil + mutable bool +} - archives := make(map[string]archive.Archive) - extract := make(map[string]map[string][]deb.ExtractInfo) - pathInfos := make(map[string]setup.PathInfo) - report := NewReport(options.TargetDir) +type contentChecker struct { + knownPaths map[string]pathData +} - knownPaths := make(map[string]bool) - knownPaths["/"] = true +func (cc *contentChecker) checkMutable(path string) error { + if !cc.knownPaths[path].mutable { + return fmt.Errorf("cannot write file which is not mutable: %s", path) + } + return nil +} - addKnownPath := func(path string) { - if path[0] != '/' { - panic("bug: tried to add relative path to known paths") - } - cleanPath := filepath.Clean(path) - slashPath := cleanPath - if path[len(path)-1] == '/' && cleanPath != "/" { - slashPath += "/" - } - for { - if _, ok := knownPaths[slashPath]; ok { - break +func (cc *contentChecker) checkKnown(path string) error { + var err error + if _, ok := cc.knownPaths[path]; !ok { + // We assume that path is clean and ends with slash if it designates a directory. + if path[len(path)-1] == '/' { + if path == "/" { + panic("internal error: content root (\"/\") is not selected") } - knownPaths[slashPath] = true - cleanPath = filepath.Dir(cleanPath) - if cleanPath == "/" { - break + if _, ok := cc.knownPaths[path[:len(path)-1]]; ok { + err = fmt.Errorf("content is not a directory: %s", path[:len(path)-1]) + } else { + err = fmt.Errorf("cannot list directory which is not selected: %s", path) + } + } else { + if _, ok := cc.knownPaths[path+"/"]; ok { + err = fmt.Errorf("content is not a file: %s", path) + } else { + err = fmt.Errorf("cannot read file which is not selected: %s", path) } - slashPath = cleanPath + "/" } } + return err +} +func Run(options *RunOptions) (*Report, error) { oldUmask := syscall.Umask(0) defer func() { syscall.Umask(oldUmask) }() targetDir := filepath.Clean(options.TargetDir) - targetDirAbs := targetDir - if !filepath.IsAbs(targetDirAbs) { + if !filepath.IsAbs(targetDir) { dir, err := os.Getwd() if err != nil { return nil, fmt.Errorf("cannot obtain current directory: %w", err) } - targetDirAbs = filepath.Join(dir, targetDir) + targetDir = filepath.Join(dir, targetDir) + } + + archives, err := selectPkgArchives(options.Archives, options.Selection) + if err != nil { + return nil, err } // Build information to process the selection. + extract := make(map[string]map[string][]deb.ExtractInfo) for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { - archives[slice.Package] = options.PkgArchives[slice.Package] extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } arch := archives[slice.Package].Options().Arch copyrightPath := "/usr/share/doc/" + slice.Package + "/copyright" - addKnownPath(copyrightPath) hasCopyright := false for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue } - if len(pathInfo.Arch) > 0 && !contains(pathInfo.Arch, arch) { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue } - if pathInfo.Kind != setup.GlobPath { - addKnownPath(targetPath) - } - pathInfos[targetPath] = pathInfo + if pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath { sourcePath := pathInfo.Info if sourcePath == "" { sourcePath = targetPath } extractPackage[sourcePath] = append(extractPackage[sourcePath], deb.ExtractInfo{ - Path: targetPath, + Path: targetPath, + Context: slice, }) if sourcePath == copyrightPath && targetPath == copyrightPath { hasCopyright = true } } else { + // When the content is not extracted from the package (i.e. path is + // not glob or copy), we add a ExtractInfo for the parent directory + // to preserve the permissions from the tarball where possible. targetDir := filepath.Dir(strings.TrimRight(targetPath, "/")) + "/" if targetDir == "" || targetDir == "/" { continue @@ -137,7 +152,68 @@ func Run(options *RunOptions) (*Report, error) { packages[slice.Package] = reader } - globbedPaths := make(map[string][]string) + // When creating content, record if a path is known and whether they are + // listed as until: mutate in all the slices that reference them. + knownPaths := map[string]pathData{} + addKnownPath(knownPaths, "/", pathData{}) + + report, err := NewReport(targetDir) + if err != nil { + return nil, fmt.Errorf("internal error: cannot create report: %w", err) + } + + // Creates the filesystem entry and adds it to the report. It also updates + // knownPaths with the files created. + create := func(extractInfos []deb.ExtractInfo, o *fsutil.CreateOptions) error { + entry, err := fsutil.Create(o) + if err != nil { + return err + } + // Content created was not listed in a slice contents because extractInfo + // is empty. + if len(extractInfos) == 0 { + return nil + } + + relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, targetDir)) + if o.Mode.IsDir() { + relPath = relPath + "/" + } + inSliceContents := false + until := setup.UntilMutate + mutable := false + for _, extractInfo := range extractInfos { + if extractInfo.Context == nil { + continue + } + slice, ok := extractInfo.Context.(*setup.Slice) + if !ok { + return fmt.Errorf("internal error: invalid Context of type %T in extractInfo", extractInfo.Context) + } + pathInfo, ok := slice.Contents[extractInfo.Path] + if !ok { + return fmt.Errorf("internal error: path %q not listed in slice contents", extractInfo.Path) + } + inSliceContents = true + mutable = mutable || pathInfo.Mutable + if pathInfo.Until == setup.UntilNone { + until = setup.UntilNone + } + // Do not add paths with "until: mutate". + if pathInfo.Until != setup.UntilMutate { + err := report.Add(slice, entry) + if err != nil { + return err + } + } + } + + if inSliceContents { + data := pathData{mutable: mutable, until: until} + addKnownPath(knownPaths, relPath, data) + } + return nil + } // Extract all packages, also using the selection order. for _, slice := range options.Selection.Slices { @@ -149,33 +225,7 @@ func Run(options *RunOptions) (*Report, error) { Package: slice.Package, Extract: extract[slice.Package], TargetDir: targetDir, - // Creates the filesystem entry and adds it to the report. - Create: func(extractInfo *deb.ExtractInfo, o *fsutil.CreateOptions) error { - entry, err := fsutil.Create(o) - if err != nil { - return err - } - - // We only want to keep the entries that were explicitly listed - // in the slice definition. - if extractInfo == nil { - return nil - } - if _, ok := slice.Contents[extractInfo.Path]; !ok { - return nil - } - - // Check whether the file was created because it matched a glob. - if strings.ContainsAny(extractInfo.Path, "*?") { - relPath := filepath.Clean("/" + strings.TrimLeft(o.Path, targetDir)) - if o.Mode.IsDir() { - relPath = relPath + "/" - } - globbedPaths[extractInfo.Path] = append(globbedPaths[extractInfo.Path], relPath) - addKnownPath(relPath) - } - return report.Add(slice, entry) - }, + Create: create, }) reader.Close() packages[slice.Package] = nil @@ -188,93 +238,43 @@ func Run(options *RunOptions) (*Report, error) { done := make(map[string]bool) for _, slice := range options.Selection.Slices { arch := archives[slice.Package].Options().Arch - for targetPath, pathInfo := range slice.Contents { - if len(pathInfo.Arch) > 0 && !contains(pathInfo.Arch, arch) { + for relPath, pathInfo := range slice.Contents { + if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue } - if done[targetPath] || pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath { + if done[relPath] || pathInfo.Kind == setup.CopyPath || pathInfo.Kind == setup.GlobPath { continue } - done[targetPath] = true - targetPath = filepath.Join(targetDir, targetPath) - targetMode := pathInfo.Mode - if targetMode == 0 { - if pathInfo.Kind == setup.DirPath { - targetMode = 0755 - } else { - targetMode = 0644 - } + done[relPath] = true + data := pathData{ + until: pathInfo.Until, + mutable: pathInfo.Mutable, } - - // Leverage tar handling of mode bits. - tarHeader := tar.Header{Mode: int64(targetMode)} - var fileContent io.Reader - var linkTarget string - switch pathInfo.Kind { - case setup.TextPath: - tarHeader.Typeflag = tar.TypeReg - fileContent = bytes.NewBufferString(pathInfo.Info) - case setup.DirPath: - tarHeader.Typeflag = tar.TypeDir - case setup.SymlinkPath: - tarHeader.Typeflag = tar.TypeSymlink - linkTarget = pathInfo.Info - default: - return nil, fmt.Errorf("internal error: cannot extract path of kind %q", pathInfo.Kind) - } - - entry, err := fsutil.Create(&fsutil.CreateOptions{ - Path: targetPath, - Mode: tarHeader.FileInfo().Mode(), - Data: fileContent, - Link: linkTarget, - MakeParents: true, - }) + addKnownPath(knownPaths, relPath, data) + targetPath := filepath.Join(targetDir, relPath) + entry, err := createFile(targetPath, pathInfo) if err != nil { return nil, err } - err = report.Add(slice, entry) - if err != nil { - return nil, err + + // Do not add paths with "until: mutate". + if pathInfo.Until != setup.UntilMutate { + err = report.Add(slice, entry) + if err != nil { + return nil, err + } } } } // Run mutation scripts. Order is fundamental here as // dependencies must run before dependents. - checkWrite := func(path string) error { - if !pathInfos[path].Mutable { - return fmt.Errorf("cannot write file which is not mutable: %s", path) - } - return nil - } - checkRead := func(path string) error { - var err error - if !knownPaths[path] { - // we assume that path is clean and ends with slash if it designates a directory - if path[len(path)-1] == '/' { - if path == "/" { - panic("internal error: content root (\"/\") is not selected") - } - if knownPaths[path[:len(path)-1]] { - err = fmt.Errorf("content is not a directory: %s", path[:len(path)-1]) - } else { - err = fmt.Errorf("cannot list directory which is not selected: %s", path) - } - } else { - if knownPaths[path+"/"] { - err = fmt.Errorf("content is not a file: %s", path) - } else { - err = fmt.Errorf("cannot read file which is not selected: %s", path) - } - } - } - return err - } + checker := contentChecker{knownPaths} content := &scripts.ContentValue{ - RootDir: targetDirAbs, - CheckWrite: checkWrite, - CheckRead: checkRead, + RootDir: targetDir, + CheckWrite: checker.checkMutable, + CheckRead: checker.checkKnown, + OnWrite: report.Mutate, } for _, slice := range options.Selection.Slices { opts := scripts.RunOptions{ @@ -290,46 +290,154 @@ func Run(options *RunOptions) (*Report, error) { } } + err = removeAfterMutate(targetDir, knownPaths) + if err != nil { + return nil, err + } + + return report, nil +} + +// removeAfterMutate removes entries marked with until: mutate. A path is marked +// only when all slices that refer to the path mark it with until: mutate. +func removeAfterMutate(rootDir string, knownPaths map[string]pathData) error { var untilDirs []string - for targetPath, pathInfo := range pathInfos { - if pathInfo.Until == setup.UntilMutate { - var targetPaths []string - if pathInfo.Kind == setup.GlobPath { - targetPaths = globbedPaths[targetPath] - } else { - targetPaths = []string{targetPath} - } - for _, targetPath := range targetPaths { - realPath, err := content.RealPath(targetPath, scripts.CheckRead) - if err == nil { - if strings.HasSuffix(targetPath, "/") { - untilDirs = append(untilDirs, realPath) - } else { - err = os.Remove(realPath) - } - } - if err != nil { - return nil, fmt.Errorf("cannot perform 'until' removal: %w", err) - } + for path, data := range knownPaths { + if data.until != setup.UntilMutate { + continue + } + realPath := filepath.Join(rootDir, path) + if strings.HasSuffix(path, "/") { + untilDirs = append(untilDirs, realPath) + } else { + err := os.Remove(realPath) + if err != nil { + return fmt.Errorf("cannot perform 'until' removal: %w", err) } } } + // Order the directories so the deepest ones appear first, this way we can + // check for empty directories properly. + sort.Slice(untilDirs, func(i, j int) bool { + return untilDirs[i] > untilDirs[j] + }) for _, realPath := range untilDirs { err := os.Remove(realPath) // The non-empty directory error is caught by IsExist as well. if err != nil && !os.IsExist(err) { - return nil, fmt.Errorf("cannot perform 'until' removal: %#v", err) + return fmt.Errorf("cannot perform 'until' removal: %#v", err) } } + return nil +} - return report, nil +// addKnownPath adds a path with its data to the list of known paths. Then it +// records that the parent directories of the path are also known. +func addKnownPath(knownPaths map[string]pathData, path string, data pathData) { + if !strings.HasPrefix(path, "/") { + panic("bug: tried to add relative path to known paths") + } + cleanPath := filepath.Clean(path) + slashPath := cleanPath + if strings.HasSuffix(path, "/") && cleanPath != "/" { + slashPath += "/" + } + for { + if _, ok := knownPaths[slashPath]; ok { + break + } + knownPaths[slashPath] = data + // The parents have empty data. + data = pathData{} + cleanPath = filepath.Dir(cleanPath) + if cleanPath == "/" { + break + } + slashPath = cleanPath + "/" + } } -func contains(l []string, s string) bool { - for _, si := range l { - if si == s { - return true +func createFile(targetPath string, pathInfo setup.PathInfo) (*fsutil.Entry, error) { + targetMode := pathInfo.Mode + if targetMode == 0 { + if pathInfo.Kind == setup.DirPath { + targetMode = 0755 + } else { + targetMode = 0644 + } + } + + // Leverage tar handling of mode bits. + tarHeader := tar.Header{Mode: int64(targetMode)} + var fileContent io.Reader + var linkTarget string + switch pathInfo.Kind { + case setup.TextPath: + tarHeader.Typeflag = tar.TypeReg + fileContent = bytes.NewBufferString(pathInfo.Info) + case setup.DirPath: + tarHeader.Typeflag = tar.TypeDir + case setup.SymlinkPath: + tarHeader.Typeflag = tar.TypeSymlink + linkTarget = pathInfo.Info + default: + return nil, fmt.Errorf("internal error: cannot extract path of kind %q", pathInfo.Kind) + } + + return fsutil.Create(&fsutil.CreateOptions{ + Path: targetPath, + Mode: tarHeader.FileInfo().Mode(), + Data: fileContent, + Link: linkTarget, + MakeParents: true, + }) +} + +// selectPkgArchives selects the highest priority archive containing the +// package, unless a particular archive is pinned within the slice definition +// file. It returns a map of archives indexed by package names. +func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { + sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) + for _, archive := range selection.Release.Archives { + if archive.Priority < 0 { + // Ignore negative priority archives unless a package specifically + // asks for it with the "archive" field. + continue + } + sortedArchives = append(sortedArchives, archive) + } + slices.SortFunc(sortedArchives, func(a, b *setup.Archive) int { + return b.Priority - a.Priority + }) + + pkgArchives := make(map[string]archive.Archive) + for _, s := range selection.Slices { + if _, ok := pkgArchives[s.Package]; ok { + continue + } + pkg := selection.Release.Packages[s.Package] + + var candidates []*setup.Archive + if pkg.Archive == "" { + candidates = sortedArchives + } else { + candidates = []*setup.Archive{selection.Release.Archives[pkg.Archive]} + } + + // If the package has not pinned any archive, choose the highest + // priority archive in which the package exists. + var chosen archive.Archive + for _, archiveInfo := range candidates { + archive := archives[archiveInfo.Name] + if archive != nil && archive.Exists(pkg.Name) { + chosen = archive + break + } + } + if chosen == nil { + return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } + pkgArchives[pkg.Name] = chosen } - return false + return pkgArchives, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index fe781197..76299b5e 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path/filepath" + "sort" "strings" . "gopkg.in/check.v1" @@ -26,16 +27,12 @@ type slicerTest struct { summary string arch string release map[string]string - pkgs map[string][]byte + archives map[string]*testArchive slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) filesystem map[string]string - // TODO: - // The results of the report do not conform to the planned implementation - // yet. Namely: - // * We do not track removed directories or changes done in Starlark. - report map[string]string - error string + report map[string]string + error string } var packageEntries = map[string][]testutil.TarEntry{ @@ -193,8 +190,12 @@ var slicerTests = []slicerTest{{ }, { summary: "Create new file using glob and preserve parent directory permissions", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.PackageData["test-package"], + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "test-package": testutil.PackageData["test-package"], + }, + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -251,9 +252,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Copyright is installed", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string][]byte{ - // Add the copyright entries to the package. - "test-package": testutil.MustMakeDeb(append(testutil.TestPackageEntries, testPackageCopyrightEntries...)), + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + // Add the copyright entries to the package. + "test-package": testutil.MustMakeDeb(append(testutil.TestPackageEntries, testPackageCopyrightEntries...)), + }, + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -282,9 +287,13 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, {"other-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.PackageData["test-package"], - "other-package": testutil.PackageData["other-package"], + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "test-package": testutil.PackageData["test-package"], + "other-package": testutil.PackageData["other-package"], + }, + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -322,52 +331,18 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"implicit-parent", "myslice"}, {"explicit-dir", "myslice"}}, - pkgs: map[string][]byte{ - "implicit-parent": testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file", "random"), - }), - "explicit-dir": testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(01777, "./dir/"), - }), - }, - release: map[string]string{ - "slices/mydir/implicit-parent.yaml": ` - package: implicit-parent - slices: - myslice: - contents: - /dir/file: - `, - "slices/mydir/explicit-dir.yaml": ` - package: explicit-dir - slices: - myslice: - contents: - /dir/: - `, - }, - filesystem: map[string]string{ - "/dir/": "dir 01777", - "/dir/file": "file 0644 a441b15f", - }, - report: map[string]string{ - "/dir/": "dir 01777 {explicit-dir_myslice}", - "/dir/file": "file 0644 a441b15f {implicit-parent_myslice}", - }, -}, { - summary: "Install two packages, explicit path has preference over implicit parent (reverse order)", - slices: []setup.SliceKey{ - {"explicit-dir", "myslice"}, - {"implicit-parent", "myslice"}}, - pkgs: map[string][]byte{ - "implicit-parent": testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file", "random"), - }), - "explicit-dir": testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(01777, "./dir/"), - }), + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "implicit-parent": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0755, "./dir/"), + testutil.Reg(0644, "./dir/file", "random"), + }), + "explicit-dir": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(01777, "./dir/"), + }), + }, + }, }, release: map[string]string{ "slices/mydir/implicit-parent.yaml": ` @@ -398,9 +373,13 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, {"other-package", "myslice"}}, - pkgs: map[string][]byte{ - "test-package": testutil.PackageData["test-package"], - "other-package": testutil.PackageData["other-package"], + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "test-package": testutil.PackageData["test-package"], + "other-package": testutil.PackageData["other-package"], + }, + }, }, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -446,7 +425,7 @@ var slicerTests = []slicerTest{{ "/dir/text-file": "file 0644 d98cf53e", }, report: map[string]string{ - "/dir/text-file": "file 0644 5b41362b {test-package_myslice}", + "/dir/text-file": "file 0644 5b41362b d98cf53e {test-package_myslice}", }, }, { summary: "Script: read a file", @@ -472,7 +451,7 @@ var slicerTests = []slicerTest{{ }, report: map[string]string{ "/dir/text-file-1": "file 0644 5b41362b {test-package_myslice}", - "/foo/text-file-2": "file 0644 d98cf53e {test-package_myslice}", + "/foo/text-file-2": "file 0644 d98cf53e 5b41362b {test-package_myslice}", }, }, { summary: "Script: use 'until' to remove file after mutate", @@ -496,9 +475,7 @@ var slicerTests = []slicerTest{{ "/foo/text-file-2": "file 0644 5b41362b", }, report: map[string]string{ - // TODO this path needs to be removed from the report. - "/dir/text-file-1": "file 0644 5b41362b {test-package_myslice}", - "/foo/text-file-2": "file 0644 d98cf53e {test-package_myslice}", + "/foo/text-file-2": "file 0644 d98cf53e 5b41362b {test-package_myslice}", }, }, { summary: "Script: use 'until' to remove wildcard after mutate", @@ -517,14 +494,7 @@ var slicerTests = []slicerTest{{ "/dir/": "dir 0755", "/other-dir/": "dir 0755", }, - report: map[string]string{ - // TODO These first three entries should be removed from the report. - "/dir/nested/": "dir 0755 {test-package_myslice}", - "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", - "/dir/nested/other-file": "file 0644 6b86b273 {test-package_myslice}", - - "/other-dir/text-file": "file 0644 5b41362b {test-package_myslice}", - }, + report: map[string]string{}, }, { summary: "Script: 'until' does not remove non-empty directories", slices: []setup.SliceKey{{"test-package", "myslice"}}, @@ -544,9 +514,29 @@ var slicerTests = []slicerTest{{ "/dir/nested/file-copy": "file 0644 cc55e2ec", }, report: map[string]string{ - "/dir/nested/": "dir 0755 {test-package_myslice}", "/dir/nested/file-copy": "file 0644 cc55e2ec {test-package_myslice}", }, +}, { + summary: "Script: writing same contents to existing file does not set the final hash in report", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/text-file: {text: data1, mutable: true} + mutate: | + content.write("/dir/text-file", "data1") + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/text-file": "file 0644 5b41362b", + }, + report: map[string]string{ + "/dir/text-file": "file 0644 5b41362b {test-package_myslice}", + }, }, { summary: "Script: cannot write non-mutable files", slices: []setup.SliceKey{{"test-package", "myslice"}}, @@ -562,6 +552,35 @@ var slicerTests = []slicerTest{{ `, }, error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, +}, { + summary: "Script: cannot write to unlisted file", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + mutate: | + content.write("/dir/text-file", "data") + `, + }, + error: `slice test-package_myslice: cannot write file which is not mutable: /dir/text-file`, +}, { + summary: "Script: cannot write to directory", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/: {make: true} + mutate: | + content.write("/dir/", "data") + `, + }, + error: `slice test-package_myslice: cannot write file which is not mutable: /dir/`, }, { summary: "Script: cannot read unlisted content", slices: []setup.SliceKey{{"test-package", "myslice2"}}, @@ -602,9 +621,10 @@ var slicerTests = []slicerTest{{ slices: myslice: contents: - /dir/text-file: {text: data1} + /dir/text-file: {text: data1, mutable: true} mutate: | content.read("/dir/text-file") + content.write("/dir/text-file", "data2") `, }, hackopt: func(c *C, opts *slicer.RunOptions) { @@ -694,9 +714,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Duplicate copyright symlink is ignored", slices: []setup.SliceKey{{"copyright-symlink-openssl", "bins"}}, - pkgs: map[string][]byte{ - "copyright-symlink-openssl": testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), - "copyright-symlink-libssl3": testutil.MustMakeDeb(packageEntries["copyright-symlink-libssl3"]), + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "copyright-symlink-openssl": testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), + "copyright-symlink-libssl3": testutil.MustMakeDeb(packageEntries["copyright-symlink-libssl3"]), + }, + }, }, release: map[string]string{ "slices/mydir/copyright-symlink-libssl3.yaml": ` @@ -755,8 +779,91 @@ var slicerTests = []slicerTest{{ }, error: `slice test-package_myslice: content is not a file: /x/y`, }, { - summary: "Non-default archive", + summary: "Multiple archives with priority", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"other-package", "myslice"}}, + archives: map[string]*testArchive{ + "foo": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + }, + }, + "bar": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from bar"), + }), + "other-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./other-file", "from bar"), + }), + }, + }, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [main] + default: true + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + "slices/mydir/other-package.yaml": ` + package: other-package + slices: + myslice: + contents: + /other-file: + `, + }, + filesystem: map[string]string{ + // The notion of "default" is obsolete and highest priority is selected + "/file": "file 0644 7a3e00f5", + // Fetched from archive "bar" as no other archive has the package. + "/other-file": "file 0644 fa0c9cdb", + }, + report: map[string]string{ + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + "/other-file": "file 0644 fa0c9cdb {other-package_myslice}", + }, +}, { + summary: "Pinned non-default archive", slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "foo": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + }, + }, + "bar": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from bar"), + }), + }, + }, + }, release: map[string]string{ "chisel.yaml": ` format: chisel-v1 @@ -765,10 +872,12 @@ var slicerTests = []slicerTest{{ version: 22.04 components: [main, universe] default: true + priority: 20 v1-public-keys: [test-key] bar: version: 22.04 components: [main] + priority: 10 v1-public-keys: [test-key] v1-public-keys: test-key: @@ -781,20 +890,456 @@ var slicerTests = []slicerTest{{ slices: myslice: contents: - /dir/nested/file: + /file: `, }, filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/nested/": "dir 0755", - "/dir/nested/file": "file 0644 84237a05", + // test-package fetched from pinned archive "bar". + "/file": "file 0644 fa0c9cdb", }, report: map[string]string{ - "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", + "/file": "file 0644 fa0c9cdb {test-package_myslice}", + }, +}, { + summary: "Pinned archive does not have the package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "foo": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + }, + }, + "bar": { + pkgs: map[string][]byte{}, + }, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + default: true + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [main] + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + archive: bar + slices: + myslice: + contents: + /file: + `, + }, + // Although archive "foo" does have the package, since archive "bar" has + // been pinned in the slice definition, no other archives will be checked. + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "No archives have the package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "foo": { + pkgs: map[string][]byte{}, + }, + "bar": { + pkgs: map[string][]byte{}, + }, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + default: true + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [main] + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "Negative priority archives are ignored when not explicitly pinned in package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "foo": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + }, + }, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + default: true + priority: -20 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + // Although test-package exists in archive "foo", the archive was ignored + // due to having a negative priority. + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "Negative priority archive explicitly pinned in package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "foo": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + }, + }, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + default: true + priority: -20 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + archive: foo + slices: + myslice: + contents: + /file: + `, + }, + filesystem: map[string]string{ + "/file": "file 0644 7a3e00f5", + }, + report: map[string]string{ + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + }, +}, { + summary: "Multiple slices of same package", + slices: []setup.SliceKey{ + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/file: + /dir/file-copy: {copy: /dir/file} + /other-dir/file: {symlink: ../dir/file} + /dir/foo/bar/: {make: true, mode: 01777} + myslice2: + contents: + /dir/other-file: + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/file-copy": "file 0644 cc55e2ec", + "/dir/foo/": "dir 0755", + "/dir/foo/bar/": "dir 01777", + "/dir/other-file": "file 0644 63d5dd49", + "/other-dir/": "dir 0755", + "/other-dir/file": "symlink ../dir/file", + }, + report: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_myslice1}", + "/dir/file-copy": "file 0644 cc55e2ec {test-package_myslice1}", + "/dir/foo/bar/": "dir 01777 {test-package_myslice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_myslice2}", + "/other-dir/file": "symlink ../dir/file {test-package_myslice1}", + }, +}, { + summary: "Same glob in several entries with until:mutate and reading from script", + slices: []setup.SliceKey{ + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/**: {until: mutate} + mutate: | + content.read("/dir/file") + myslice2: + contents: + /dir/**: + mutate: | + content.read("/dir/file") + `, + }, + filesystem: map[string]string{ + "/dir/nested/other-file": "file 0644 6b86b273", + "/dir/several/": "dir 0755", + "/dir/several/levels/": "dir 0755", + "/dir/several/levels/deep/file": "file 0644 6bc26dff", + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/nested/": "dir 0755", + "/dir/nested/file": "file 0644 84237a05", + "/dir/other-file": "file 0644 63d5dd49", + "/dir/several/levels/deep/": "dir 0755", + }, + report: map[string]string{ + "/dir/": "dir 0755 {test-package_myslice2}", + "/dir/file": "file 0644 cc55e2ec {test-package_myslice2}", + "/dir/nested/": "dir 0755 {test-package_myslice2}", + "/dir/nested/file": "file 0644 84237a05 {test-package_myslice2}", + "/dir/nested/other-file": "file 0644 6b86b273 {test-package_myslice2}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_myslice2}", + "/dir/several/": "dir 0755 {test-package_myslice2}", + "/dir/several/levels/": "dir 0755 {test-package_myslice2}", + "/dir/several/levels/deep/": "dir 0755 {test-package_myslice2}", + "/dir/several/levels/deep/file": "file 0644 6bc26dff {test-package_myslice2}", + }, +}, { + summary: "Overlapping globs, until:mutate and reading from script", + slices: []setup.SliceKey{ + {"test-package", "myslice2"}, + {"test-package", "myslice1"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/**: + mutate: | + content.read("/dir/file") + myslice2: + contents: + /dir/nested/**: {until: mutate} + mutate: | + content.read("/dir/file") + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/nested/": "dir 0755", + "/dir/nested/file": "file 0644 84237a05", + "/dir/nested/other-file": "file 0644 6b86b273", + "/dir/other-file": "file 0644 63d5dd49", + "/dir/several/": "dir 0755", + "/dir/several/levels/": "dir 0755", + "/dir/several/levels/deep/": "dir 0755", + "/dir/several/levels/deep/file": "file 0644 6bc26dff", + }, + report: map[string]string{ + "/dir/": "dir 0755 {test-package_myslice1}", + "/dir/file": "file 0644 cc55e2ec {test-package_myslice1}", + "/dir/nested/": "dir 0755 {test-package_myslice1}", + "/dir/nested/file": "file 0644 84237a05 {test-package_myslice1}", + "/dir/nested/other-file": "file 0644 6b86b273 {test-package_myslice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_myslice1}", + "/dir/several/": "dir 0755 {test-package_myslice1}", + "/dir/several/levels/": "dir 0755 {test-package_myslice1}", + "/dir/several/levels/deep/": "dir 0755 {test-package_myslice1}", + "/dir/several/levels/deep/file": "file 0644 6bc26dff {test-package_myslice1}", + }, +}, { + summary: "Overlapping glob and single entry, until:mutate on entry and reading from script", + slices: []setup.SliceKey{ + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/**: + mutate: | + content.read("/dir/file") + myslice2: + contents: + /dir/file: {until: mutate} + mutate: | + content.read("/dir/file") + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/nested/": "dir 0755", + "/dir/nested/file": "file 0644 84237a05", + "/dir/nested/other-file": "file 0644 6b86b273", + "/dir/other-file": "file 0644 63d5dd49", + "/dir/several/": "dir 0755", + "/dir/several/levels/": "dir 0755", + "/dir/several/levels/deep/": "dir 0755", + "/dir/several/levels/deep/file": "file 0644 6bc26dff", + }, + report: map[string]string{ + "/dir/": "dir 0755 {test-package_myslice1}", + "/dir/file": "file 0644 cc55e2ec {test-package_myslice1}", + "/dir/nested/": "dir 0755 {test-package_myslice1}", + "/dir/nested/file": "file 0644 84237a05 {test-package_myslice1}", + "/dir/nested/other-file": "file 0644 6b86b273 {test-package_myslice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_myslice1}", + "/dir/several/": "dir 0755 {test-package_myslice1}", + "/dir/several/levels/": "dir 0755 {test-package_myslice1}", + "/dir/several/levels/deep/": "dir 0755 {test-package_myslice1}", + "/dir/several/levels/deep/file": "file 0644 6bc26dff {test-package_myslice1}", + }, +}, { + summary: "Overlapping glob and single entry, until:mutate on glob and reading from script", + slices: []setup.SliceKey{ + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/**: {until: mutate} + mutate: | + content.read("/dir/file") + myslice2: + contents: + /dir/file: + mutate: | + content.read("/dir/file") + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + }, + report: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_myslice2}", + }, +}, { + summary: "Overlapping glob and single entry, until:mutate on both and reading from script", + slices: []setup.SliceKey{ + {"test-package", "myslice1"}, + {"test-package", "myslice2"}, + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice1: + contents: + /dir/**: {until: mutate} + mutate: | + content.read("/dir/file") + myslice2: + contents: + /dir/file: {until: mutate} + mutate: | + content.read("/dir/file") + `, + }, + filesystem: map[string]string{}, + report: map[string]string{}, +}, { + summary: "Relative paths are properly trimmed during extraction", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "test-package": testutil.MustMakeDeb([]testutil.TarEntry{ + // This particular path starting with "/foo" is chosen to test for + // a particular bug; which appeared due to the usage of + // strings.TrimLeft() instead strings.TrimPrefix() to determine a + // relative path. Since TrimLeft takes in a cutset instead of a + // prefix, the desired relative path was not produced. + // See https://github.com/canonical/chisel/pull/145. + testutil.Dir(0755, "./foo-bar/"), + }), + }, + }, + }, + hackopt: func(c *C, opts *slicer.RunOptions) { + opts.TargetDir = filepath.Join(filepath.Clean(opts.TargetDir), "foo") + err := os.Mkdir(opts.TargetDir, 0755) + c.Assert(err, IsNil) + }, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /foo-bar/: + mutate: | + content.list("/foo-bar/") + `, }, }, { summary: "Pro archives", slices: []setup.SliceKey{{"test-package", "myslice"}}, + archives: map[string]*testArchive{ + "ubuntu": {}, + "fips": {}, + "fips-updates": {}, + "apps": {}, + "infra": {}, + }, release: map[string]string{ "chisel.yaml": ` format: chisel-v1 @@ -870,12 +1415,12 @@ var defaultChiselYaml = ` ` type testArchive struct { - options archive.Options + options *archive.Options pkgs map[string][]byte } func (a *testArchive) Options() *archive.Options { - return &a.options + return a.options } func (a *testArchive) Fetch(pkg string) (io.ReadCloser, error) { @@ -913,92 +1458,86 @@ func (s *S) TestRun(c *C) { func runSlicerTests(c *C, tests []slicerTest) { for _, test := range tests { - c.Logf("Summary: %s", test.summary) - - if _, ok := test.release["chisel.yaml"]; !ok { - test.release["chisel.yaml"] = string(defaultChiselYaml) - } + for _, slices := range testutil.Permutations(test.slices) { + c.Logf("Summary: %s", test.summary) - if test.pkgs == nil { - test.pkgs = map[string][]byte{ - "test-package": testutil.PackageData["test-package"], + if _, ok := test.release["chisel.yaml"]; !ok { + test.release["chisel.yaml"] = string(defaultChiselYaml) } - } - releaseDir := c.MkDir() - for path, data := range test.release { - fpath := filepath.Join(releaseDir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0755) - c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0644) - c.Assert(err, IsNil) - } + if test.archives == nil { + test.archives = map[string]*testArchive{ + "ubuntu": { + pkgs: map[string][]byte{ + "test-package": testutil.PackageData["test-package"], + }, + }, + } + } + for _, archive := range test.archives { + if archive.pkgs == nil { + archive.pkgs = map[string][]byte{ + "test-package": testutil.PackageData["test-package"], + } + } + } - release, err := setup.ReadRelease(releaseDir) - c.Assert(err, IsNil) + releaseDir := c.MkDir() + for path, data := range test.release { + fpath := filepath.Join(releaseDir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + c.Assert(err, IsNil) + } - selection, err := setup.Select(release, test.slices) - c.Assert(err, IsNil) + release, err := setup.ReadRelease(releaseDir) + c.Assert(err, IsNil) - archives := map[string]archive.Archive{} - for name, setupArchive := range release.Archives { - archive := &testArchive{ - options: archive.Options{ - Label: setupArchive.Name, - Version: setupArchive.Version, - Suites: setupArchive.Suites, - Components: setupArchive.Components, - Pro: setupArchive.Pro, - Arch: test.arch, - }, - pkgs: test.pkgs, - } - archives[name] = archive - } + selection, err := setup.Select(release, slices) + c.Assert(err, IsNil) - pkgArchives := map[string]archive.Archive{} - for _, pkg := range release.Packages { - if pkg.Archive == "" { - var chosen *setup.Archive - for _, releaseArchive := range selection.Release.Archives { - archive := archives[releaseArchive.Name] - if archive == nil || !archive.Exists(pkg.Name) { - continue - } - if chosen == nil || chosen.Priority < releaseArchive.Priority { - chosen = releaseArchive + archives := map[string]archive.Archive{} + for name, setupArchive := range release.Archives { + testArchive, ok := test.archives[name] + c.Assert(ok, Equals, true) + if testArchive.options == nil { + testArchive.options = &archive.Options{ + Label: setupArchive.Name, + Version: setupArchive.Version, + Suites: setupArchive.Suites, + Components: setupArchive.Components, + Pro: setupArchive.Pro, + Arch: test.arch, } } - c.Assert(chosen, NotNil) - pkgArchives[pkg.Name] = archives[chosen.Name] - } else { - pkgArchives[pkg.Name] = archives[pkg.Archive] + archives[name] = testArchive } - } - targetDir := c.MkDir() - options := slicer.RunOptions{ - Selection: selection, - PkgArchives: pkgArchives, - TargetDir: targetDir, - } - if test.hackopt != nil { - test.hackopt(c, &options) - } - report, err := slicer.Run(&options) - if test.error == "" { - c.Assert(err, IsNil) - } else { - c.Assert(err, ErrorMatches, test.error) - continue - } + targetDir := c.MkDir() + options := slicer.RunOptions{ + Selection: selection, + Archives: archives, + TargetDir: targetDir, + } + if test.hackopt != nil { + test.hackopt(c, &options) + } + report, err := slicer.Run(&options) + if test.error == "" { + c.Assert(err, IsNil) + } else { + c.Assert(err, ErrorMatches, test.error) + continue + } - if test.filesystem != nil { - c.Assert(testutil.TreeDump(targetDir), DeepEquals, test.filesystem) - } + if test.filesystem != nil { + c.Assert(testutil.TreeDump(targetDir), DeepEquals, test.filesystem) + } - if test.report != nil { - c.Assert(treeDumpReport(report), DeepEquals, test.report) + if test.report != nil { + c.Assert(treeDumpReport(report), DeepEquals, test.report) + } } } } @@ -1021,6 +1560,8 @@ func treeDumpReport(report *slicer.Report) map[string]string { case 0: // Regular if entry.Size == 0 { fsDump = fmt.Sprintf("file %#o empty", entry.Mode.Perm()) + } else if entry.FinalHash != "" { + fsDump = fmt.Sprintf("file %#o %s %s", fperm, entry.Hash[:8], entry.FinalHash[:8]) } else { fsDump = fmt.Sprintf("file %#o %s", fperm, entry.Hash[:8]) } @@ -1033,6 +1574,7 @@ func treeDumpReport(report *slicer.Report) map[string]string { for slice := range entry.Slices { slicesStr = append(slicesStr, slice.String()) } + sort.Strings(slicesStr) result[entry.Path] = fmt.Sprintf("%s {%s}", fsDump, strings.Join(slicesStr, ",")) } return result diff --git a/internal/strdist/strdist.go b/internal/strdist/strdist.go index eb0c1dca..f205bfcf 100644 --- a/internal/strdist/strdist.go +++ b/internal/strdist/strdist.go @@ -102,8 +102,8 @@ func Distance(a, b string, f CostFunc, cut int64) int64 { // Supported wildcards: // // ? - Any one character, except for / -// * - Any zero or more characters, execept for / -// ** - Any zero or more characrers, including / +// * - Any zero or more characters, except for / +// ** - Any zero or more characters, including / func GlobPath(a, b string) bool { a = strings.ReplaceAll(a, "**", "⁑") b = strings.ReplaceAll(b, "**", "⁑") diff --git a/internal/testutil/permutation.go b/internal/testutil/permutation.go new file mode 100644 index 00000000..11817b1c --- /dev/null +++ b/internal/testutil/permutation.go @@ -0,0 +1,34 @@ +package testutil + +func Permutations[S ~[]E, E any](s S) []S { + var output []S + // Heap's algorithm: https://en.wikipedia.org/wiki/Heap%27s_algorithm. + var generate func(k int, s S) + generate = func(k int, s S) { + if k <= 1 { + r := make([]E, len(s)) + copy(r, s) + output = append(output, r) + return + } + // Generate permutations with k-th unaltered. + // Initially k = length(A). + generate(k-1, s) + + // Generate permutations for k-th swapped with each k-1 initial. + for i := 0; i < k-1; i += 1 { + // Swap choice dependent on parity of k (even or odd). + if k%2 == 0 { + s[i], s[k-1] = s[k-1], s[i] + } else { + s[0], s[k-1] = s[k-1], s[0] + } + generate(k-1, s) + } + } + + sCpy := make([]E, len(s)) + copy(sCpy, s) + generate(len(sCpy), sCpy) + return output +} diff --git a/internal/testutil/permutation_test.go b/internal/testutil/permutation_test.go new file mode 100644 index 00000000..af2afb48 --- /dev/null +++ b/internal/testutil/permutation_test.go @@ -0,0 +1,73 @@ +package testutil_test + +import ( + "sort" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/testutil" +) + +type permutationSuite struct{} + +var _ = Suite(&permutationSuite{}) + +var permutationTests = []struct { + slice []any + res [][]any +}{ + { + slice: []any{}, + res: [][]any{{}}, + }, + { + slice: []any{1}, + res: [][]any{{1}}, + }, + { + slice: []any{1, 2}, + res: [][]any{{1, 2}, {2, 1}}, + }, + { + slice: []any{1, 2, 3}, + res: [][]any{{1, 2, 3}, {2, 1, 3}, {3, 1, 2}, {1, 3, 2}, {2, 3, 1}, {3, 2, 1}}, + }, +} + +func (*permutationSuite) TestPermutations(c *C) { + for _, test := range permutationTests { + c.Assert(testutil.Permutations(test.slice), DeepEquals, test.res) + } +} + +func (*permutationSuite) TestFuzzPermutations(c *C) { + for sLen := 0; sLen <= 10; sLen++ { + s := make([]byte, sLen) + for i := 0; i < sLen; i++ { + s[i] = byte(i) + } + permutations := testutil.Permutations(s) + + // Factorial. + expectedLen := 1 + for i := 2; i <= len(s); i++ { + expectedLen *= i + } + c.Assert(len(permutations), Equals, expectedLen) + + duplicated := map[string]bool{} + for _, perm := range permutations { + // []byte is not comparable. + permStr := string(perm) + if _, ok := duplicated[permStr]; ok { + c.Fatalf("duplicated permutation: %v", perm) + } + duplicated[permStr] = true + // Check that the elements are the same. + sort.Slice(perm, func(i, j int) bool { + return perm[i] < perm[j] + }) + c.Assert(perm, DeepEquals, s, Commentf("invalid elements in permutation %v of base slice %v", perm, s)) + } + } +} diff --git a/internal/testutil/treedump.go b/internal/testutil/treedump.go index 72457887..26aa164d 100644 --- a/internal/testutil/treedump.go +++ b/internal/testutil/treedump.go @@ -6,6 +6,8 @@ import ( "io/fs" "os" "path/filepath" + + "github.com/canonical/chisel/internal/fsutil" ) func TreeDump(dir string) map[string]string { @@ -60,3 +62,25 @@ func TreeDump(dir string) map[string]string { } return result } + +// TreeDumpEntry the file information in the same format as [testutil.TreeDump]. +func TreeDumpEntry(entry *fsutil.Entry) string { + fperm := entry.Mode.Perm() + if entry.Mode&fs.ModeSticky != 0 { + fperm |= 01000 + } + switch entry.Mode.Type() { + case fs.ModeDir: + return fmt.Sprintf("dir %#o", fperm) + case fs.ModeSymlink: + return fmt.Sprintf("symlink %s", entry.Link) + case 0: // Regular + if entry.Size == 0 { + return fmt.Sprintf("file %#o empty", entry.Mode.Perm()) + } else { + return fmt.Sprintf("file %#o %s", fperm, entry.Hash[:8]) + } + default: + panic(fmt.Errorf("unknown file type %d: %s", entry.Mode.Type(), entry.Path)) + } +} diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index f1bc6e78..ee1c284c 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -34,7 +34,7 @@ parts: plugin: go source: . build-snaps: - - go/1.20/stable + - go/1.21/stable build-environment: - CGO_ENABLED: 0 - GOFLAGS: -trimpath -ldflags=-w -ldflags=-s diff --git a/spread.yaml b/spread.yaml index 70a18942..510025a1 100644 --- a/spread.yaml +++ b/spread.yaml @@ -40,7 +40,7 @@ backends: discard: docker rm -f $SPREAD_SYSTEM systems: - - ubuntu-23.04: + - ubuntu-24.04: username: ubuntu password: ubuntu diff --git a/tests/find/task.yaml b/tests/find/task.yaml new file mode 100644 index 00000000..283d7bc3 --- /dev/null +++ b/tests/find/task.yaml @@ -0,0 +1,14 @@ +summary: Chisel can find slice by slice name, package name or a combination + +execute: | + find() { + fullname=$1 + shift + query=$@ + chisel find --release ${OS}-${RELEASE} $query | grep $fullname + } + + find "ca-certificates_data" "ca-certificates_data" + find "ca-certificates_data" "ca-certificates" "_data" + find "ca-certificates_data" "_data" "ca-certificates" + ! find "ca-certificates_data" "ca-certificates" "foo" From d1bafd9b5948aa3e0a598247ac2590ab263228b8 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 7 Aug 2024 09:12:01 +0600 Subject: [PATCH 05/60] fix: export ProValue and errors on invalid pro values Export ProValue type and related constants: ProNone, ProFIPS, ProFIPSUpdates, ProApps, ProInfra. Additionally, raise errors on invalid "pro" values. --- internal/setup/setup.go | 19 +++++++++---------- internal/setup/setup_test.go | 29 +++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index ef604fcd..3d44bc42 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -353,14 +353,14 @@ type yamlRelease struct { V1PubKeys map[string]yamlPubKey `yaml:"v1-public-keys"` } -type proValue string +type ProValue string const ( - proNone proValue = "" - proFIPS proValue = "fips" - proFIPSUpdates proValue = "fips-updates" - proApps proValue = "apps" - proInfra proValue = "infra" + ProNone ProValue = "" + ProFIPS ProValue = "fips" + ProFIPSUpdates ProValue = "fips-updates" + ProApps ProValue = "apps" + ProInfra ProValue = "infra" ) const ( @@ -374,7 +374,7 @@ type yamlArchive struct { Components []string `yaml:"components"` Default bool `yaml:"default"` Priority int `yaml:"priority"` - Pro proValue `yaml:"pro"` + Pro ProValue `yaml:"pro"` PubKeys []string `yaml:"public-keys"` // V1PubKeys is used for compatibility with format "chisel-v1". V1PubKeys []string `yaml:"v1-public-keys"` @@ -516,10 +516,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { release.DefaultArchive = archiveName } switch details.Pro { - case proNone, proApps, proFIPS, proFIPSUpdates, proInfra: + case ProNone, ProApps, ProFIPS, ProFIPSUpdates, ProInfra: default: - logf("%s: archive %q ignored due to invalid pro value: %s", fileName, archiveName, details.Pro) - continue + return nil, fmt.Errorf("%s: archive %q has invalid pro value: %s", fileName, archiveName, details.Pro) } if len(details.PubKeys) == 0 { if yamlVar.Format == "chisel-v1" { diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 9da5a9f9..2ffc53b9 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1360,7 +1360,7 @@ var setupTests = []setupTest{{ }, relerror: `slices/test-package.yaml: package refers to undefined archive "non-existing"`, }, { - summary: "Pro values in archives", + summary: "Valid Pro values in archives", input: map[string]string{ "chisel.yaml": ` format: chisel-v1 @@ -1399,13 +1399,6 @@ var setupTests = []setupTest{{ pro: infra priority: 15 v1-public-keys: [test-key] - foo: - version: 20.04 - components: [main] - suites: [foo] - pro: foo - priority: -10 - v1-public-keys: [test-key] v1-public-keys: test-key: id: ` + testKey.ID + ` @@ -1470,6 +1463,26 @@ var setupTests = []setupTest{{ }, }, }, +}, { + summary: "Invalid Pro value in archive", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 20.04 + components: [main] + suites: [foo] + pro: unknown-value + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + }, + relerror: `chisel.yaml: archive "foo" has invalid pro value: unknown-value`, }} var defaultChiselYaml = ` From 6479279608ef837ba34452f5a66c8cee99a8f8bf Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 7 Aug 2024 12:35:05 +0600 Subject: [PATCH 06/60] fix: remove redundant type, map and function --- internal/archive/archive.go | 53 +++++++++--------------- internal/archive/archive_test.go | 35 ++++------------ internal/archive/export_test.go | 2 + internal/setup/setup.go | 22 +++++----- internal/slicer/slicer_test.go | 69 -------------------------------- 5 files changed, 39 insertions(+), 142 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 2637e628..7b1907af 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -14,6 +14,7 @@ import ( "github.com/canonical/chisel/internal/control" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/pgputil" + "github.com/canonical/chisel/internal/setup" ) type Archive interface { @@ -43,12 +44,6 @@ func Open(options *Options) (Archive, error) { if err != nil { return nil, err } - if options.Pro != "" { - err = validatePro(options.Pro) - if err != nil { - return nil, err - } - } return openUbuntu(options) } @@ -139,26 +134,28 @@ const ubuntuURL = "http://archive.ubuntu.com/ubuntu/" const ubuntuPortsURL = "http://ports.ubuntu.com/ubuntu-ports/" var proArchiveInfo = map[string]struct { - baseURL, label string + BaseURL, Label string }{ - "fips": { - baseURL: "https://esm.ubuntu.com/fips/ubuntu/", - label: "UbuntuFIPS", + setup.ProFIPS: { + BaseURL: "https://esm.ubuntu.com/fips/ubuntu/", + Label: "UbuntuFIPS", }, - "fips-updates": { - baseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", - label: "UbuntuFIPSUpdates", + setup.ProFIPSUpdates: { + BaseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", + Label: "UbuntuFIPSUpdates", }, - "apps": { - baseURL: "https://esm.ubuntu.com/apps/ubuntu/", - label: "UbuntuESMApps", + setup.ProApps: { + BaseURL: "https://esm.ubuntu.com/apps/ubuntu/", + Label: "UbuntuESMApps", }, - "infra": { - baseURL: "https://esm.ubuntu.com/infra/ubuntu/", - label: "UbuntuESM", + setup.ProInfra: { + BaseURL: "https://esm.ubuntu.com/infra/ubuntu/", + Label: "UbuntuESM", }, } +// archiveURL returns the archive base URL depending on the "pro" value and +// selected architecture "arch". func archiveURL(pro, arch string) string { if pro == "" { if arch == "amd64" || arch == "i386" { @@ -166,14 +163,7 @@ func archiveURL(pro, arch string) string { } return ubuntuPortsURL } - return proArchiveInfo[pro].baseURL -} - -func validatePro(pro string) error { - if _, ok := proArchiveInfo[pro]; !ok { - return fmt.Errorf("unknown pro value: %s", pro) - } - return nil + return proArchiveInfo[pro].BaseURL } func openUbuntu(options *Options) (Archive, error) { @@ -193,13 +183,10 @@ func openUbuntu(options *Options) (Archive, error) { Dir: options.CacheDir, }, pubKeys: options.PubKeys, + baseURL: archiveURL(options.Pro, options.Arch), } - if archive.baseURL == "" { - archive.baseURL = archiveURL(options.Pro, options.Arch) - } - - if options.Pro != "" && archive.creds == nil { + if options.Pro != "" { creds, err := findCredentials(archive.baseURL) if err != nil { return nil, err @@ -279,7 +266,7 @@ func (index *ubuntuIndex) fetchRelease() error { // Parse the appropriate section for the type of archive. label := "Ubuntu" if index.archive.options.Pro != "" { - label = proArchiveInfo[index.archive.options.Pro].label + label = proArchiveInfo[index.archive.options.Pro].Label } section := ctrl.Section(label) if section == nil { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index ad9c77bb..41b52656 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -322,7 +322,7 @@ func (s *httpSuite) TestArchiveLabels(c *C) { _, err = archive.Open(&options) c.Assert(err, IsNil) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("ThirdParty")) + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("Unknown")) options = archive.Options{ Label: "ubuntu", @@ -335,28 +335,7 @@ func (s *httpSuite) TestArchiveLabels(c *C) { } _, err = archive.Open(&options) - c.Assert(err, ErrorMatches, `.*\bno Ubuntu section`) -} - -var proArchiveInfo = map[string]struct { - baseURL, label string -}{ - "fips": { - baseURL: "https://esm.ubuntu.com/fips/ubuntu/", - label: "UbuntuFIPS", - }, - "fips-updates": { - baseURL: "https://esm.ubuntu.com/fips-updates/ubuntu/", - label: "UbuntuFIPSUpdates", - }, - "apps": { - baseURL: "https://esm.ubuntu.com/apps/ubuntu/", - label: "UbuntuESMApps", - }, - "infra": { - baseURL: "https://esm.ubuntu.com/infra/ubuntu/", - label: "UbuntuESM", - }, + c.Assert(err, ErrorMatches, `corrupted archive InRelease file: no Ubuntu section`) } func (s *httpSuite) TestProArchives(c *C) { @@ -372,15 +351,15 @@ func (s *httpSuite) TestProArchives(c *C) { confFile := filepath.Join(credsDir, "credentials") contents := "" - for _, info := range proArchiveInfo { - contents += fmt.Sprintf("machine %s login foo password bar\n", info.baseURL) + for _, info := range archive.ProArchiveInfo { + contents += fmt.Sprintf("machine %s login foo password bar\n", info.BaseURL) } err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) - for pro, info := range proArchiveInfo { - s.base = info.baseURL - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel(info.label)) + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel(info.Label)) options := archive.Options{ Label: "ubuntu", diff --git a/internal/archive/export_test.go b/internal/archive/export_test.go index cd75d5f4..3569a699 100644 --- a/internal/archive/export_test.go +++ b/internal/archive/export_test.go @@ -19,3 +19,5 @@ type Credentials = credentials var FindCredentials = findCredentials var FindCredentialsInDir = findCredentialsInDir + +var ProArchiveInfo = proArchiveInfo diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 3d44bc42..09ef7484 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -27,6 +27,14 @@ type Release struct { DefaultArchive string } +const ( + ProNone = "" + ProFIPS = "fips" + ProFIPSUpdates = "fips-updates" + ProApps = "apps" + ProInfra = "infra" +) + // Archive is the location from which binary packages are obtained. type Archive struct { Name string @@ -353,16 +361,6 @@ type yamlRelease struct { V1PubKeys map[string]yamlPubKey `yaml:"v1-public-keys"` } -type ProValue string - -const ( - ProNone ProValue = "" - ProFIPS ProValue = "fips" - ProFIPSUpdates ProValue = "fips-updates" - ProApps ProValue = "apps" - ProInfra ProValue = "infra" -) - const ( MaxArchivePriority = 1000 MinArchivePriority = -1000 @@ -374,7 +372,7 @@ type yamlArchive struct { Components []string `yaml:"components"` Default bool `yaml:"default"` Priority int `yaml:"priority"` - Pro ProValue `yaml:"pro"` + Pro string `yaml:"pro"` PubKeys []string `yaml:"public-keys"` // V1PubKeys is used for compatibility with format "chisel-v1". V1PubKeys []string `yaml:"v1-public-keys"` @@ -543,7 +541,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { Version: details.Version, Suites: details.Suites, Components: details.Components, - Pro: string(details.Pro), + Pro: details.Pro, Priority: details.Priority, PubKeys: archiveKeys, } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 3c3bb0a3..ff977f82 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1330,75 +1330,6 @@ var slicerTests = []slicerTest{{ content.list("/foo-bar/") `, }, -}, { - summary: "Pro archives", - slices: []setup.SliceKey{{"test-package", "myslice"}}, - archives: map[string]*testArchive{ - "ubuntu": {}, - "fips": {}, - "fips-updates": {}, - "apps": {}, - "infra": {}, - }, - release: map[string]string{ - "chisel.yaml": ` - format: chisel-v1 - archives: - ubuntu: - version: 20.04 - components: [main] - suites: [focal] - priority: 10 - v1-public-keys: [test-key] - fips: - version: 20.04 - components: [main] - suites: [focal] - pro: fips - priority: 20 - v1-public-keys: [test-key] - fips-updates: - version: 20.04 - components: [main] - suites: [focal-updates] - pro: fips-updates - priority: 21 - v1-public-keys: [test-key] - apps: - version: 20.04 - components: [main] - suites: [focal-apps-security] - pro: apps - priority: 16 - v1-public-keys: [test-key] - infra: - version: 20.04 - components: [main] - suites: [focal-infra-security] - pro: infra - priority: 15 - v1-public-keys: [test-key] - v1-public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` - `, - "slices/mydir/test-package.yaml": ` - package: test-package - slices: - myslice: - contents: - /dir/nested/file: - `, - }, - filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/nested/": "dir 0755", - "/dir/nested/file": "file 0644 84237a05", - }, - report: map[string]string{ - "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", - }, }, { summary: "Multiple slices of same package", slices: []setup.SliceKey{ From cd3dd754c09f4c312f39ab1f97ab24af914a46b9 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 7 Aug 2024 13:20:11 +0600 Subject: [PATCH 07/60] doc: update comment for archiveURL --- internal/archive/archive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 7b1907af..fd073353 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -155,7 +155,7 @@ var proArchiveInfo = map[string]struct { } // archiveURL returns the archive base URL depending on the "pro" value and -// selected architecture "arch". +// selected architecture "arch". The "pro" value should be validated. func archiveURL(pro, arch string) string { if pro == "" { if arch == "amd64" || arch == "i386" { From f23cca6f3f36ae5c5d54f45668a8958faceeb888 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 8 Aug 2024 09:35:41 +0600 Subject: [PATCH 08/60] fix: warn on invalid archive pro value, no errors --- internal/setup/setup.go | 6 +++++- internal/setup/setup_test.go | 21 ++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 09ef7484..e3e6a37b 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -516,7 +516,8 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { switch details.Pro { case ProNone, ProApps, ProFIPS, ProFIPSUpdates, ProInfra: default: - return nil, fmt.Errorf("%s: archive %q has invalid pro value: %s", fileName, archiveName, details.Pro) + logf("warning: %s: archive %q ignored due to invalid pro value: %s", fileName, archiveName, details.Pro) + continue } if len(details.PubKeys) == 0 { if yamlVar.Format == "chisel-v1" { @@ -546,6 +547,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { PubKeys: archiveKeys, } } + if len(release.Archives) == 0 { + return nil, fmt.Errorf("%s: no valid archives defined", fileName) + } return release, err } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 2ffc53b9..07fed44f 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1399,6 +1399,13 @@ var setupTests = []setupTest{{ pro: infra priority: 15 v1-public-keys: [test-key] + foo: + version: 20.04 + components: [main] + suites: [foo] + pro: unknown-value + priority: 10 + v1-public-keys: [test-key] v1-public-keys: test-key: id: ` + testKey.ID + ` @@ -1464,25 +1471,21 @@ var setupTests = []setupTest{{ }, }, }, { - summary: "Invalid Pro value in archive", + summary: "No valid archives defined", input: map[string]string{ "chisel.yaml": ` format: chisel-v1 archives: - foo: + invalid: version: 20.04 components: [main] - suites: [foo] - pro: unknown-value + suites: [focal] priority: 10 v1-public-keys: [test-key] - v1-public-keys: - test-key: - id: ` + testKey.ID + ` - armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + pro: unknown-value `, }, - relerror: `chisel.yaml: archive "foo" has invalid pro value: unknown-value`, + relerror: `chisel.yaml: no valid archives defined`, }} var defaultChiselYaml = ` From 5c676fc0096f690b003fd8a5ce3a72738d9babbb Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 8 Aug 2024 09:36:12 +0600 Subject: [PATCH 09/60] refactor: archive baseURL selection --- internal/archive/archive.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index fd073353..c18b2450 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -157,13 +157,13 @@ var proArchiveInfo = map[string]struct { // archiveURL returns the archive base URL depending on the "pro" value and // selected architecture "arch". The "pro" value should be validated. func archiveURL(pro, arch string) string { - if pro == "" { - if arch == "amd64" || arch == "i386" { - return ubuntuURL - } - return ubuntuPortsURL + if pro != "" { + return proArchiveInfo[pro].BaseURL + } + if arch == "amd64" || arch == "i386" { + return ubuntuURL } - return proArchiveInfo[pro].BaseURL + return ubuntuPortsURL } func openUbuntu(options *Options) (Archive, error) { From 025acdab43743fe49f111b1a9cb10626c650e956 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 8 Aug 2024 13:37:48 +0600 Subject: [PATCH 10/60] fix: log pro value with archive label --- internal/archive/archive.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index c18b2450..f1155f6a 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -229,7 +229,7 @@ func openUbuntu(options *Options) (Archive, error) { } func (index *ubuntuIndex) fetchRelease() error { - logf("Fetching %s %s %s suite details...", index.label, index.version, index.suite) + logf("Fetching %s %s %s suite details...", index.proSuffixedLabel(), index.version, index.suite) reader, err := index.fetch("InRelease", "", fetchDefault) if err != nil { return err @@ -286,7 +286,7 @@ func (index *ubuntuIndex) fetchIndex() error { return fmt.Errorf("%s is missing from %s %s component digests", packagesPath, index.suite, index.component) } - logf("Fetching index for %s %s %s %s component...", index.label, index.version, index.suite, index.component) + logf("Fetching index for %s %s %s %s component...", index.proSuffixedLabel(), index.version, index.suite, index.component) reader, err := index.fetch(packagesPath+".gz", digest, fetchBulk) if err != nil { return err @@ -384,3 +384,13 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea return index.archive.cache.Open(writer.Digest()) } + +// proSuffixedLabel adds " (pro)" suffix to the label and returns it +// if the archive is specified with pro value. Otherwise, it returns the +// original label. +func (index *ubuntuIndex) proSuffixedLabel() string { + if index.archive.options.Pro == "" { + return index.label + } + return index.label + " " + index.archive.options.Pro + " (pro)" +} From 706941ecaa6904647f2bf66cb4e0dadc01cb9f05 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 8 Aug 2024 13:38:18 +0600 Subject: [PATCH 11/60] fix: remove ProNone value, use "" --- internal/setup/setup.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index e3e6a37b..b00821f1 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -28,7 +28,6 @@ type Release struct { } const ( - ProNone = "" ProFIPS = "fips" ProFIPSUpdates = "fips-updates" ProApps = "apps" @@ -514,9 +513,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { release.DefaultArchive = archiveName } switch details.Pro { - case ProNone, ProApps, ProFIPS, ProFIPSUpdates, ProInfra: + case "", ProApps, ProFIPS, ProFIPSUpdates, ProInfra: default: - logf("warning: %s: archive %q ignored due to invalid pro value: %s", fileName, archiveName, details.Pro) + logf("Ignoring archive %q (invalid pro value: %s)...", archiveName, details.Pro) continue } if len(details.PubKeys) == 0 { From 6d15dd17d2344b1971c4ff60c5fbadb83dc6656b Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 8 Aug 2024 14:05:02 +0600 Subject: [PATCH 12/60] fix: do not fetch index if arch is unsupported Do not fetch indexes for a particular suite if the suite repository does not support architecture packages. The "Architectures" field in the InRelease/Release file should specify the list of supported architectures. --- internal/archive/archive.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index f1155f6a..3c78d64f 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "slices" "strings" "time" @@ -212,6 +213,11 @@ func openUbuntu(options *Options) (Archive, error) { return nil, err } release = index.release + if !index.supportsArch(options.Arch) { + // Release does not support the specified architecture. Do + // not add indexes of this release. + break + } err = index.checkComponents(options.Components) if err != nil { return nil, err @@ -300,6 +306,11 @@ func (index *ubuntuIndex) fetchIndex() error { return nil } +func (index *ubuntuIndex) supportsArch(arch string) bool { + supportedArches := strings.Fields(index.release.Get("Architectures")) + return slices.Contains(supportedArches, arch) +} + func (index *ubuntuIndex) checkComponents(components []string) error { releaseComponents := strings.Fields(index.release.Get("Components")) for _, c1 := range components { From 60319265872e6a1008f3f0ec6f1053276fec075e Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Fri, 9 Aug 2024 15:48:28 +0600 Subject: [PATCH 13/60] fix: add warnings when ignoring archive suite/component --- internal/archive/archive.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 3c78d64f..195c1868 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -216,6 +216,7 @@ func openUbuntu(options *Options) (Archive, error) { if !index.supportsArch(options.Arch) { // Release does not support the specified architecture. Do // not add indexes of this release. + logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...", index.proSuffixedLabel(), index.version, index.suite, options.Arch) break } err = index.checkComponents(options.Components) From a0948d00f21d777f6655bed6c548dd10fa8aa65f Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Fri, 9 Aug 2024 17:48:03 +0600 Subject: [PATCH 14/60] test(spread): add test for pro archives --- .github/workflows/spread.yml | 2 + spread.yaml | 1 + .../pro-archives/chisel-releases/chisel.yaml | 45 +++++++++++++++++++ .../chisel-releases/slices/hello.yaml | 13 ++++++ tests/pro-archives/task.yaml | 22 +++++++++ 5 files changed, 83 insertions(+) create mode 100644 tests/pro-archives/chisel-releases/chisel.yaml create mode 100644 tests/pro-archives/chisel-releases/slices/hello.yaml create mode 100644 tests/pro-archives/task.yaml diff --git a/.github/workflows/spread.yml b/.github/workflows/spread.yml index 190a9061..ae95b7a1 100644 --- a/.github/workflows/spread.yml +++ b/.github/workflows/spread.yml @@ -27,6 +27,8 @@ jobs: go-version: '>=1.17.0' - name: Build and run spread + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} run: | (cd _spread/cmd/spread && go build) _spread/cmd/spread/spread -v focal jammy mantic noble diff --git a/spread.yaml b/spread.yaml index 510025a1..598a7399 100644 --- a/spread.yaml +++ b/spread.yaml @@ -4,6 +4,7 @@ path: /chisel environment: OS: ubuntu + PRO_TOKEN: $(HOST:echo $PRO_TOKEN) backends: # Cannot use LXD backend due to https://github.com/snapcore/spread/issues/154 diff --git a/tests/pro-archives/chisel-releases/chisel.yaml b/tests/pro-archives/chisel-releases/chisel.yaml new file mode 100644 index 00000000..7de9d8c5 --- /dev/null +++ b/tests/pro-archives/chisel-releases/chisel.yaml @@ -0,0 +1,45 @@ +format: v1 + +archives: + ubuntu: + version: 24.04 + pro: infra + components: [main] + suites: [noble-infra-security, noble-infra-updates] + public-keys: [ubuntu-esm-key-v2] + +public-keys: + # Ubuntu Extended Security Maintenance Automatic Signing Key v2 + # rsa4096/56f7650a24c9e9ecf87c4d8d4067e40313cb4b13 2019-04-17T02:33:33Z + ubuntu-esm-key-v2: + id: "4067E40313CB4B13" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY + 680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs + 8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t + YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 + t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI + OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG + JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 + rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 + K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT + Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE + 6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB + tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj + IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 + kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C + NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB + m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S + nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 + 1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F + EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC + puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG + tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C + DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS + 89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 + RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi + JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc + =KWLs + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/pro-archives/chisel-releases/slices/hello.yaml b/tests/pro-archives/chisel-releases/slices/hello.yaml new file mode 100644 index 00000000..7bd1a393 --- /dev/null +++ b/tests/pro-archives/chisel-releases/slices/hello.yaml @@ -0,0 +1,13 @@ +package: hello + +essential: + - hello_copyright + +slices: + bins: + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: diff --git a/tests/pro-archives/task.yaml b/tests/pro-archives/task.yaml new file mode 100644 index 00000000..262baab9 --- /dev/null +++ b/tests/pro-archives/task.yaml @@ -0,0 +1,22 @@ +summary: Chisel can fetch packages from Ubuntu Pro archives + +variants: + - noble + +environment: + ROOTFS: rootfs + +prepare: | + apt update && apt install -y ubuntu-pro-client + pro attach ${PRO_TOKEN} --no-auto-enable + pro enable esm-infra --assume-yes + mkdir ${ROOTFS} + +restore: | + pro detach --assume-yes + rm -r ${ROOTFS} + +execute: | + chisel cut --release ./chisel-releases/ --root ${ROOTFS} hello_bins + test -f ${ROOTFS}/usr/bin/hello + test -f ${ROOTFS}/usr/share/doc/hello/copyright From c2753a5a173663bd4874604b0559d842907b0685 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 13 Aug 2024 11:29:08 +0600 Subject: [PATCH 15/60] test: refactor real archive tests --- internal/archive/archive_test.go | 68 ++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 41b52656..a0835fc4 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -440,37 +440,57 @@ func (s *S) TestRealArchive(c *C) { if !*realArchiveFlag { c.Skip("--real-archive not provided") } - for _, release := range ubuntuReleases { + for _, test := range realArchiveTests { for _, arch := range elfToDebArch { - s.testOpenArchiveArch(c, release, arch) + s.testOpenArchiveArch(c, test, arch) } } } -type ubuntuRelease struct { +type realArchiveTest struct { name string version string + suites []string + components []string archivePubKeys []*packet.PublicKey + pkg string + binPath string + copyrightText string } -var ubuntuReleases = []ubuntuRelease{{ - name: "focal", - version: "20.04", +var realArchiveTests = []realArchiveTest{{ + name: "focal", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main", "universe"}, archivePubKeys: []*packet.PublicKey{ keyUbuntu2018.PubKey, }, + pkg: "hostname", + binPath: "/bin/hostname", + copyrightText: "This package was written by Peter Tobias ", }, { - name: "jammy", - version: "22.04", + name: "jammy", + version: "22.04", + suites: []string{"jammy"}, + components: []string{"main", "universe"}, archivePubKeys: []*packet.PublicKey{ keyUbuntu2018.PubKey, }, + pkg: "hostname", + binPath: "/bin/hostname", + copyrightText: "This package was written by Peter Tobias ", }, { - name: "noble", - version: "24.04", + name: "noble", + version: "24.04", + suites: []string{"noble"}, + components: []string{"main", "universe"}, archivePubKeys: []*packet.PublicKey{ keyUbuntu2018.PubKey, }, + pkg: "hostname", + binPath: "/usr/bin/hostname", + copyrightText: "This package was written by Peter Tobias ", }} var elfToDebArch = map[elf.Machine]string{ @@ -492,17 +512,17 @@ func (s *S) checkArchitecture(c *C, arch string, binaryPath string) { c.Assert(binaryArch, Equals, arch) } -func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { - c.Logf("Checking ubuntu archive %s %s...", release.name, arch) +func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { + c.Logf("Checking ubuntu archive %s %s...", test.name, arch) options := archive.Options{ Label: "ubuntu", - Version: release.version, + Version: test.version, Arch: arch, - Suites: []string{release.name}, - Components: []string{"main", "universe"}, + Suites: test.suites, + Components: test.components, CacheDir: c.MkDir(), - PubKeys: release.archivePubKeys, + PubKeys: test.archivePubKeys, } archive, err := archive.Open(&options) @@ -510,18 +530,18 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { extractDir := c.MkDir() - pkg, err := archive.Fetch("hostname") + pkg, err := archive.Fetch(test.pkg) c.Assert(err, IsNil) err = deb.Extract(pkg, &deb.ExtractOptions{ - Package: "hostname", + Package: test.pkg, TargetDir: extractDir, Extract: map[string][]deb.ExtractInfo{ - "/usr/share/doc/hostname/copyright": { + fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, - "/bin/hostname": { - {Path: "/hostname"}, + test.binPath: { + {Path: "/binary"}, }, }, }) @@ -529,9 +549,7 @@ func (s *S) testOpenArchiveArch(c *C, release ubuntuRelease, arch string) { data, err := os.ReadFile(filepath.Join(extractDir, "copyright")) c.Assert(err, IsNil) + c.Assert(strings.Contains(string(data), test.copyrightText), Equals, true) - copyrightTop := "This package was written by Peter Tobias " - c.Assert(strings.Contains(string(data), copyrightTop), Equals, true) - - s.checkArchitecture(c, arch, filepath.Join(extractDir, "hostname")) + s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) } From c1f44d969dab30e7963732d44e0024d25f9d0b45 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 13 Aug 2024 15:02:39 +0600 Subject: [PATCH 16/60] feat: add gpg keys for fips, esm-apps, esm-infra --- internal/testutil/pgpkeys.go | 117 +++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/internal/testutil/pgpkeys.go b/internal/testutil/pgpkeys.go index e999a9a3..98eb6b05 100644 --- a/internal/testutil/pgpkeys.go +++ b/internal/testutil/pgpkeys.go @@ -21,6 +21,18 @@ var PGPKeys = map[string]*PGPKeyData{ ID: "871920D1991BC93C", PubKeyArmor: pubKeyUbuntu2018Armor, }, + "key-ubuntu-fips-v1": { + ID: "C1997C40EDE22758", + PubKeyArmor: pubKeyUbuntuFIPSv1Armor, + }, + "key-ubuntu-apps": { + ID: "AB01A101DB53907B", + PubKeyArmor: pubKeyUbuntuAppsArmor, + }, + "key-ubuntu-esm-v2": { + ID: "4067E40313CB4B13", + PubKeyArmor: pubKeyUbuntuESMv2Armor, + }, "key1": { ID: "854BAF1AA9D76600", PubKeyArmor: pubKey1Armor, @@ -87,6 +99,111 @@ uOgcXny1UlwtCUzlrSaP -----END PGP PUBLIC KEY BLOCK----- ` +// Ubuntu Federal Information Processing Standards Automatic Signing Key V1 . +// ID: C1997C40EDE22758. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuFIPSv1Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFzZxGABEADSWmX0+K//0cosKPyr5m1ewmwWKjRo/KBPTyR8icHhbBWfFd8T +DtYggvQHPU0YnKRcWits0et8JqSgZttNa28s7SaSUTBzfgzFJZgULAi/4i8u8TUj ++KH2zSoUX55NKC9aozba1cR66jM6O/BHXK5YoZzTpmiY1AHlIWAJ9s6cCClhnYMR +zwxSZVbefcYFbVPX/dQw/FMvJVeZ2aQ18NDgMQciu786aYklMFowxWNs/eLLTqum +cDHaw9UpKyhgfL/mkaIXuhYy6YRByYq1oOnJ5XffAOtovvCti1MvsPc0NDhPiGLf +9Fd/GtnqHxzVDqZmtUXX50mGu4LnJoHgWRjml3mapDPStzFr7Xgbb0NnyflmxnfN +kQcu2lFyXFfndWwg/RAOFdBPxBQhRK52uZiCfydKD7zCXz9YGm9xEK541EG0FrwA +6Vk1xaFol/jI8MQdP1o3JySX0Pqva3IHF7FHWHmxrIPaJLIHi0IrFG6Fgmk4sQ2w +XSc8kbxR+wYYKqIhBUZP0eb1jkFfvRVS6YvAy18xtw5pFD+VURdA0Uu5cotESfyz +oHsQ5R7wzg76oV/mYukHGC0x8peqxiPwbyhGFAhG8eUR66iYZgGbzmNI+OJz2EUi +UZJJXt4rnI1RVJLbhK9RjeobkOjf58Cm8RExlqJU16gy9saCMSiAqHx8swARAQAB +tFxVYnVudHUgRmVkZXJhbCBJbmZvcm1hdGlvbiBQcm9jZXNzaW5nIFN0YW5kYXJk +cyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgVjEgPGVzbUBjYW5vbmljYWwuY29tPokC +OAQTAQIAIgUCXNnEYAIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQwZl8 +QO3iJ1j4Vw//SawfmZi1GW+EUnuPqSz+zcmIKdx6AWZTe9/vSj6jgq4SYt//LAiD +NQz3dn2m0m5AaCucza2BCixUBrNhMh66m+lXfTqymUtTIpWpu4L1WLUbPjQ+s3Ad +xuF7S5wJtQrYmPvmZduZgg1wcb8eaqVltRJREpOP6sxcuqtvcfv4v4QYZ+iYd7eJ +8fxPOiyJEOTQPTdPZahYTaUOIloN5pT6uVg03u59Kh4aHCYxlRorvuRBabdctCfA +EBgomk4Us20Tv31dqlvMAiGKJqf1wdjhzlUmk4g/fOiRSNETKSC/VeUGH0fSbizl +Gs7Mg60jChPKpwzB6Rb5Nv2/Aw/FlSkfFhMdCdfKjl8IWOMPmElTVJFyVx1mmURi +3LgsloDFmJfebXefSFA7S8KLyBGlZJ/APaym64Ls12PUOjfh1Glie3E8KO66AGLo +ID1dQnzRizuHxW80ET03dSjzTXHLSi+iFycmNAxo6gB3GyOQ8tlIHjo1FfDfNYDf +qKic3Q0B9TvF6hqVRIcyePK4lN5YtRpVRdVj/jv8AqbzaIaVCP4k4nNrbaVx5zQf +BWq2E9IH+vLZfPyiP+hwxswfrlU3mrXBpPStIxq41yXFwQiDnqgkhEVAcrYPjBnS +T6s3+b+4HbAW6mbp4jEHUd/F1+iXz90T2WArrNIkMbmpChMuSyRN8Hc= +=DWhM +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Apps Automatic Signing Key . +// ID: AB01A101DB53907B. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuAppsArmor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF3WVA4BEAC7MDr8HClfKptSd4VeB12Vy+Ao/4NpY2ITdkRed4vfh/4eBWWn +3+in6So2ekweifACSxScB/M9zVObsI1cab7QPMkIiATNUfIyOEP7iNWLX4+AytM1 +LP3bZo8OpghnLZNstCGbiRUO4CDNmCI04DOPCu9EVEO4WWNuWIMRwCLShDSf7Cid +J2fn2TT/7vsmA4eI3YnAne+u8g4X2zMHQFkHANhylB0lPyThXo5jaxHImzm4wf/2 +LF8f1Y1nRQObS2jcvYc3fm9B7iOGpyNAw3h6hrPKH5T9tY/ZoMtFHqn66J1CBSHb +hDkEvA46X50su4yAHeSiEG/hMYG7SoHzmAsjEXnvkTIE41WhmxlidQnRs2uWy34U +7VmOpaidWn3R99fNHYOtSOB6bpIvls8snWSQ63jcFXnt05nVZsp/Ixzl0Oqitynx +DFwoxEwt3ZuCHwxbx2vZ+FiZXVFN7I0IyBDOEL6XS27FNaMCZ7Q/6z/ckdWto55E +264OWf9lnw31bXFXHWSusRXWzD6FK8dqWgjtrWwRxlvF4jm688lqpjac6fFES3UK +BhjyHXFGL/+HHZ9CNxlLYF5QnXq1mGR0Ykw975u8KoOFSLBqsx+1a21m6dfzujY7 +2Gq6Sju+9Yo1aOF+CNvTMYdRBoDL4sFj6VAmUsszMA5aAb+82pOCaDvGJQARAQAB +tDVVYnVudHUgQXBwcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGVzbUBjYW5vbmlj +YWwuY29tPokCOAQTAQIAIgUCXdZUDgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC +F4AACgkQqwGhAdtTkHuTOw/8Czv42TSpwHz+eNtl3ZFyxta9rR/qWC3h+vMu0R/l +5KU3aQQOygWOoUcr1QTPSSg3v/H+v/8vqVq2UuUxSIfpMxBj2kIX2vqskv6Roez7 +xR8lVDa0a47z/NYMfKpxrEJxOLh/c7I6aAsa597bTqDHtucHL/22BvfUJJqw6jq1 +7SswP5lqKPBFz7x+E2hgfJE7Vn7h0ICm29FkWnOeTKfj8VwTAeKXKUI9Hw6+aqr9 +29Y2NdLsYZ57mpivRLNM9sBZoF3avP1pUC2k0IwP3dwh4AxUMXjRRPh173iXBfR2 +yAf1lWET/5+8dSBrfFIZSo+FF/EEBmqIVtJpHkq8+YxUbCLbkoikRi2kwrgyXLEn +FqxSU2Ab0xurFHiHcJoCGVD38xjznO5cQl7H4K9+B/rFpTTowOHbOcFpKAzpYqB5 +8rnR1yRSsB33zac8xesUIfzYWRtLc5/VIb5mOkWlb62d8emILx2XuRFVjKq6mKki +oGckhDUOuEFrjW1cQq+PWBBxyJoXcy6wGSoPJ/ELeaf9zg8SF0jwuN6BPHVBeJ/E +W53zR5iV0N9fRT+M2JN5tc5HenO92xLgPAh+GPWLYmPdTmHu+kFozqsHx/NUw2iP +PBL6Q1VZytt2Uf6qLPUx7GpYMKf42Vldb0feFo/YA/lzOgPlY29pDLKXbse6o+Sr +kmk= +=AEEr +-----END PGP PUBLIC KEY BLOCK----- +` + +// Ubuntu Extended Security Maintenance Automatic Signing Key v2 . +// ID: 4067E40313CB4B13. +// Useful to validate InRelease files from live archive. +const pubKeyUbuntuESMv2Armor = ` +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFy2kH0BEADl/2e2pULZaSRovd3E1i1cVk3zebzndHZm/hK8/Srx69ivw3pY +680gFE/N3s3R/C5Jh9ThdD1zpGmxVdqcABSPmW1FczdFZY2E37HMH7Uijs4CsnFs +8nrNGQaqX/T1g2fQqjia3zkabMeehUEZC5GPYjpeeFW6Wy1O1A1Tzu7/Wjc+uF/t +YYe/ZPXea74QZphu/N+8dy/ts/IzL2VtXuxiegGLfBFqzgZuBmlxXHVhftKvcis9 +t2ko65uVyDcLtItMhSJokKBsIYJliqOXjUbQf5dz8vLXkku94arBMgsxDWT4K/xI +OTsaI/GMlSIKQ6Ucd/GKrBEsy5O8RDtD9A2klV7YeEwPEgqL+RhpdxAs/xUeTOZG +JKwuvlBjzIhJF9bIfbyzx7DdcGFqRE+a8eBIUMQjVkt9Yk7jj0eV3oVTE7XNhb53 +rHuPL+zJVkiharxiTgYvkow3Nlbg3oURx9Ln67ni9pUtI1HbortGZsAkyOcpep58 +K9cYvUePJWzjkY+bjcGKR19CWPl7KaUalIf2Tao5OwtqjrblTsXdtV7eG45ys0MT +Kl/DeqTJ0w6+i4eq4ZUfOCL/DIwS5zUB9j1KMUgEfocjYIdHWI8TSrA8jLYNPbVE +6+WjekHMB9liNrEQoESWBddS+bglPxuVwy2paGTUYJW1GnRZOTD+CG4ETQARAQAB +tFFVYnVudHUgRXh0ZW5kZWQgU2VjdXJpdHkgTWFpbnRlbmFuY2UgQXV0b21hdGlj +IFNpZ25pbmcgS2V5IHYyIDxlc21AY2Fub25pY2FsLmNvbT6JAjgEEwECACIFAly2 +kH0CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEEBn5AMTy0sTo/8QAJ1C +NhAkZ+Xq/BZ8UzAFCQn6GlIYg/ueY216xcQdDX1uN8hNOlPTNmftroIvohFAfFtB +m5galzY3DBPU8eZr8Y8XgiGD97wkR4zfhfh1EK/6diMG/HG00kdcWquFXMRB7E7S +nDTpyuPfkAzm9n6l69UB3UA53CaEUuVJ7qFfZsWgiQeUJpvqD0MIVsWr+T/paSx7 +1JE9BVatFefq0egErv1sa2uYgcH9TRZMLw6gYxWtXeGA08Cpp0+OEvIzmJOHo5/F +EpJ3hGk87Of77BC7FbqSDpeYkcjnlI2i0QAxxFygKhPOMLuA4XVn3TDuqCgTFIFC +puupzIX/Up51FJmo64V9GZ/uF0jZy4tDxsCRJnEV+4Kv2sU5uMlmNchZMBjXYGiG +tpH9CqJkSZjFvB6bk+Ot98KI6+CuNWn1N0sXFKpEUGdJLuOKfJ9+xI5plo8Bct5C +DM9s4l0IuAPCsyayXrSmlyOAHzxDUeRMCEUnXWfycCUyqdyYIcCMPLV44Ccg9NyS +89dEauSCPuyCSxm5UYEHQdsSI/+rxRdS9IzoKs4za2L7fhY8PfdPlmghmXc/chz1 +RtgjPfAsUHUPRr0h//TzxRm5dbYdUyqMPzZcDO8wYBT/4xrwnFkSHZhnVxpw7PDi +JYK4SVVc4ZO20PE1+RZc5oSbt4hRbFTCSb31Pydc +=KWLs +-----END PGP PUBLIC KEY BLOCK----- +` + // Test-purpose RSA 2048 bits signing key-pairs without a passphrase. // ID: 854BAF1AA9D76600. User: "foo-bar ". const pubKey1Armor = ` From f84e257157dc2c3462bb4a786c3f9c98d2e6b2aa Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 13 Aug 2024 16:06:32 +0600 Subject: [PATCH 17/60] test: real archive tests for Ubuntu Pro archives To run the real archive tests for Ubuntu Pro archives, the host machine needs to be Pro-enabled first. sudo pro attach --no-auto-enable There are tests for fips, fips-updates, esm-apps, esm-infra archives. Thus, the respective services have to be enabled as well to generate the APT credentials. sudo pro enable fips fips-updates esm-apps esm-infra --assume-yes We need to pass the --pro-archive flag in addition to the --real-archive flag to run the tests for Pro archives. go test ./internal/archive/ --real-archive --pro-archive Note that the credentials file(s) in /etc/apt/auth.conf.d (or CHISEL_AUTH_DIR) may require root permission. test: pass missing pro field in archive.Options --- internal/archive/archive_test.go | 130 +++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 34 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index a0835fc4..842f7eb7 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -40,9 +40,12 @@ type httpSuite struct { var _ = Suite(&httpSuite{}) var ( - key1 = testutil.PGPKeys["key1"] - key2 = testutil.PGPKeys["key2"] - keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + key1 = testutil.PGPKeys["key1"] + key2 = testutil.PGPKeys["key2"] + keyUbuntu2018 = testutil.PGPKeys["key-ubuntu-2018"] + keyUbuntuFIPSv1 = testutil.PGPKeys["key-ubuntu-fips-v1"] + keyUbuntuApps = testutil.PGPKeys["key-ubuntu-apps"] + keyUbuntuESMv2 = testutil.PGPKeys["key-ubuntu-esm-v2"] ) func (s *httpSuite) SetUpTest(c *C) { @@ -433,15 +436,33 @@ func read(r io.Reader) string { // ---------------------------------------------------------------------------------------- // Real archive tests, only enabled via --real-archive. +// +// Additionally, run tests against Ubuntu Pro archives by passing --pro-archive. +// The host machine must be Pro enabled and relevant Pro services must be +// enabled. The following commands might help: +// sudo pro attach --no-auto-enable +// sudo pro enable fips fips-updates esm-apps esm-infra --assume-yes var realArchiveFlag = flag.Bool("real-archive", false, "Perform tests against real archive") +var proArchiveFlag = flag.Bool("pro-archive", false, "Perform tests against real Ubuntu Pro archive") func (s *S) TestRealArchive(c *C) { if !*realArchiveFlag { c.Skip("--real-archive not provided") } + allArch := make([]string, 0, len(elfToDebArch)) + for _, arch := range elfToDebArch { + allArch = append(allArch, arch) + } for _, test := range realArchiveTests { - for _, arch := range elfToDebArch { + if test.pro != "" && !*proArchiveFlag { + // --pro-archive is not provided. Ignore test. + continue + } + if len(test.architectures) == 0 { + test.architectures = allArch + } + for _, arch := range test.architectures { s.testOpenArchiveArch(c, test, arch) } } @@ -452,45 +473,85 @@ type realArchiveTest struct { version string suites []string components []string + pro string archivePubKeys []*packet.PublicKey + architectures []string pkg string binPath string copyrightText string } var realArchiveTests = []realArchiveTest{{ - name: "focal", - version: "20.04", - suites: []string{"focal"}, - components: []string{"main", "universe"}, - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, - pkg: "hostname", - binPath: "/bin/hostname", - copyrightText: "This package was written by Peter Tobias ", + name: "focal", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + binPath: "/bin/hostname", + copyrightText: "This package was written by Peter Tobias ", }, { - name: "jammy", - version: "22.04", - suites: []string{"jammy"}, - components: []string{"main", "universe"}, - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, - pkg: "hostname", - binPath: "/bin/hostname", - copyrightText: "This package was written by Peter Tobias ", + name: "jammy", + version: "22.04", + suites: []string{"jammy"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + binPath: "/bin/hostname", + copyrightText: "This package was written by Peter Tobias ", }, { - name: "noble", - version: "24.04", - suites: []string{"noble"}, - components: []string{"main", "universe"}, - archivePubKeys: []*packet.PublicKey{ - keyUbuntu2018.PubKey, - }, - pkg: "hostname", - binPath: "/usr/bin/hostname", - copyrightText: "This package was written by Peter Tobias ", + name: "noble", + version: "24.04", + suites: []string{"noble"}, + components: []string{"main", "universe"}, + archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, + pkg: "hostname", + binPath: "/usr/bin/hostname", + copyrightText: "This package was written by Peter Tobias ", +}, { + name: "fips", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main"}, + pro: "fips", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + architectures: []string{"amd64"}, + pkg: "openssh-client", + binPath: "/usr/bin/ssh", + copyrightText: "1995 Tatu Ylonen , Espoo, Finland", +}, { + name: "fips-updates", + version: "20.04", + suites: []string{"focal-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + architectures: []string{"amd64"}, + pkg: "openssh-client", + binPath: "/usr/bin/ssh", + copyrightText: "1995 Tatu Ylonen , Espoo, Finland", +}, { + name: "esm-apps", + version: "20.04", + suites: []string{"focal-apps-security", "focal-apps-updates"}, + components: []string{"main"}, + pro: "apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + architectures: []string{"amd64"}, + pkg: "hello", + binPath: "/usr/bin/hello", + copyrightText: "This package was first put together by Ian Jackson", +}, { + name: "esm-infra", + version: "20.04", + suites: []string{"focal-infra-security", "focal-infra-updates"}, + components: []string{"main"}, + pro: "infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + architectures: []string{"amd64"}, + pkg: "hello", + binPath: "/usr/bin/hello", + copyrightText: "This package was first put together by Ian Jackson", }} var elfToDebArch = map[elf.Machine]string{ @@ -522,6 +583,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { Suites: test.suites, Components: test.components, CacheDir: c.MkDir(), + Pro: test.pro, PubKeys: test.archivePubKeys, } From 64d1e748273c5498419a659b5f4d802c3b6385f3 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 3 Oct 2024 17:24:53 +0600 Subject: [PATCH 18/60] feat: support multiple archives with "priority" field (#12) This commit adds support for fetching packages from multiple archives. It introduces a new field `archives..priority` which takes in a signed integer and specifies the priority of a certain archive. A package is fetched from the archive with highest priority. It also deprecates the concept of default archive in chisel.yaml. However, the field can still be present and it will be parsed but IGNORED. DEPRECATED: "archives..default" field in chisel.yaml. --------- Co-authored-by: Alberto Carretero --- cmd/chisel/cmd_find_test.go | 2 - cmd/chisel/cmd_info_test.go | 8 - internal/setup/setup.go | 52 ++-- internal/setup/setup_test.go | 238 ++++++++++------- internal/slicer/slicer.go | 64 ++++- internal/slicer/slicer_test.go | 463 +++++++++++++++++++++++++-------- internal/testutil/archive.go | 13 +- 7 files changed, 591 insertions(+), 249 deletions(-) diff --git a/cmd/chisel/cmd_find_test.go b/cmd/chisel/cmd_find_test.go index 7d7c0d7a..a2a68c2a 100644 --- a/cmd/chisel/cmd_find_test.go +++ b/cmd/chisel/cmd_find_test.go @@ -33,8 +33,6 @@ func makeSamplePackage(pkg string, slices []string) *setup.Package { } var sampleRelease = &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", diff --git a/cmd/chisel/cmd_info_test.go b/cmd/chisel/cmd_info_test.go index c78a477c..9dcfa3d8 100644 --- a/cmd/chisel/cmd_info_test.go +++ b/cmd/chisel/cmd_info_test.go @@ -25,7 +25,6 @@ var infoTests = []infoTest{{ query: []string{"mypkg1_myslice1"}, stdout: ` package: mypkg1 - archive: ubuntu slices: myslice1: contents: @@ -37,7 +36,6 @@ var infoTests = []infoTest{{ query: []string{"mypkg2"}, stdout: ` package: mypkg2 - archive: ubuntu slices: myslice: contents: @@ -49,7 +47,6 @@ var infoTests = []infoTest{{ query: []string{"mypkg1_myslice2", "mypkg1_myslice1"}, stdout: ` package: mypkg1 - archive: ubuntu slices: myslice1: contents: @@ -65,7 +62,6 @@ var infoTests = []infoTest{{ query: []string{"mypkg1_myslice1", "mypkg2", "mypkg1_myslice2"}, stdout: ` package: mypkg1 - archive: ubuntu slices: myslice1: contents: @@ -76,7 +72,6 @@ var infoTests = []infoTest{{ - mypkg2_myslice --- package: mypkg2 - archive: ubuntu slices: myslice: contents: @@ -88,7 +83,6 @@ var infoTests = []infoTest{{ query: []string{"mypkg1_myslice1", "mypkg1"}, stdout: ` package: mypkg1 - archive: ubuntu slices: myslice1: contents: @@ -104,7 +98,6 @@ var infoTests = []infoTest{{ query: []string{"mypkg1_myslice1", "mypkg1_myslice1", "mypkg1_myslice1"}, stdout: ` package: mypkg1 - archive: ubuntu slices: myslice1: contents: @@ -121,7 +114,6 @@ var infoTests = []infoTest{{ query: []string{"foo", "mypkg1_myslice1", "bar_foo"}, stdout: ` package: mypkg1 - archive: ubuntu slices: myslice1: contents: diff --git a/internal/setup/setup.go b/internal/setup/setup.go index e604786d..c9b255d7 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -21,10 +21,9 @@ import ( // Release is a collection of package slices targeting a particular // distribution version. type Release struct { - Path string - Packages map[string]*Package - Archives map[string]*Archive - DefaultArchive string + Path string + Packages map[string]*Package + Archives map[string]*Archive } // Archive is the location from which binary packages are obtained. @@ -33,6 +32,7 @@ type Archive struct { Version string Suites []string Components []string + Priority int PubKeys []*packet.PublicKey } @@ -229,6 +229,28 @@ func (r *Release) validate() error { return err } + // Check for archive priority conflicts. + priorities := make(map[int]*Archive) + for _, archive := range r.Archives { + if old, ok := priorities[archive.Priority]; ok { + if old.Name > archive.Name { + archive, old = old, archive + } + return fmt.Errorf("chisel.yaml: archives %q and %q have the same priority value of %d", old.Name, archive.Name, archive.Priority) + } + priorities[archive.Priority] = archive + } + + // Check that archives pinned in packages are defined. + for _, pkg := range r.Packages { + if pkg.Archive == "" { + continue + } + if _, ok := r.Archives[pkg.Archive]; !ok { + return fmt.Errorf("%s: package refers to undefined archive %q", pkg.Path, pkg.Archive) + } + } + return nil } @@ -355,9 +377,6 @@ func readSlices(release *Release, baseDir, dirName string) error { if err != nil { return err } - if pkg.Archive == "" { - pkg.Archive = release.DefaultArchive - } release.Packages[pkg.Name] = pkg } @@ -372,11 +391,16 @@ type yamlRelease struct { V1PubKeys map[string]yamlPubKey `yaml:"v1-public-keys"` } +const ( + MaxArchivePriority = 1000 + MinArchivePriority = -1000 +) + type yamlArchive struct { Version string `yaml:"version"` Suites []string `yaml:"suites"` Components []string `yaml:"components"` - Default bool `yaml:"default"` + Priority int `yaml:"priority"` PubKeys []string `yaml:"public-keys"` // V1PubKeys is used for compatibility with format "chisel-v1". V1PubKeys []string `yaml:"v1-public-keys"` @@ -547,14 +571,6 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if len(details.Components) == 0 { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } - if len(yamlVar.Archives) == 1 { - details.Default = true - } else if details.Default && release.DefaultArchive != "" { - return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, release.DefaultArchive, archiveName) - } - if details.Default { - release.DefaultArchive = archiveName - } if len(details.PubKeys) == 0 { if yamlVar.Format == "chisel-v1" { return nil, fmt.Errorf("%s: archive %q missing v1-public-keys field", fileName, archiveName) @@ -570,11 +586,15 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { } archiveKeys = append(archiveKeys, key) } + if details.Priority > MaxArchivePriority || details.Priority < MinArchivePriority { + return nil, fmt.Errorf("%s: archive %q has invalid priority value of %d", fileName, archiveName, details.Priority) + } release.Archives[archiveName] = &Archive{ Name: archiveName, Version: details.Version, Suites: details.Suites, Components: details.Components, + Priority: details.Priority, PubKeys: archiveKeys, } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 2d146585..ae919dcf 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -73,8 +73,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -86,10 +84,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, @@ -117,8 +114,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -130,9 +125,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -179,8 +173,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -192,9 +184,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -440,8 +431,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -453,9 +442,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -655,8 +643,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -668,9 +654,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -695,8 +680,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -708,9 +691,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -736,8 +718,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -749,9 +729,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -766,7 +745,7 @@ var setupTests = []setupTest{{ }, }, }, { - summary: "Multiple archives", + summary: "Multiple archives with priorities", input: map[string]string{ "chisel.yaml": ` format: chisel-v1 @@ -775,12 +754,13 @@ var setupTests = []setupTest{{ version: 22.04 components: [main, universe] suites: [jammy] - default: true + priority: 20 v1-public-keys: [test-key] bar: version: 22.04 components: [universe] suites: [jammy-updates] + priority: -10 v1-public-keys: [test-key] v1-public-keys: test-key: @@ -792,14 +772,13 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "foo", - Archives: map[string]*setup.Archive{ "foo": { Name: "foo", Version: "22.04", Suites: []string{"jammy"}, Components: []string{"main", "universe"}, + Priority: 20, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, "bar": { @@ -807,18 +786,65 @@ var setupTests = []setupTest{{ Version: "22.04", Suites: []string{"jammy-updates"}, Components: []string{"universe"}, + Priority: -10, PubKeys: []*packet.PublicKey{testKey.PubKey}, }, }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "foo", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, +}, { + summary: "Two archives cannot have same priority", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + suites: [jammy] + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [universe] + suites: [jammy-updates] + priority: 20 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + relerror: `chisel.yaml: archives "bar" and "foo" have the same priority value of 20`, +}, { + summary: "Invalid archive priority", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + suites: [jammy] + priority: 10000 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + }, + relerror: `chisel.yaml: archive "foo" has invalid priority value of 10000`, }, { summary: "Extra fields in YAML are ignored (necessary for forward compatibility)", input: map[string]string{ @@ -849,8 +875,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -862,9 +886,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -888,12 +911,13 @@ var setupTests = []setupTest{{ components: [main, universe] suites: [jammy] v1-public-keys: [extra-key] - default: true + priority: 20 bar: version: 22.04 components: [universe] suites: [jammy-updates] v1-public-keys: [test-key, extra-key] + priority: 10 v1-public-keys: extra-key: id: ` + extraTestKey.ID + ` @@ -907,14 +931,13 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "foo", - Archives: map[string]*setup.Archive{ "foo": { Name: "foo", Version: "22.04", Suites: []string{"jammy"}, Components: []string{"main", "universe"}, + Priority: 20, PubKeys: []*packet.PublicKey{extraTestKey.PubKey}, }, "bar": { @@ -922,15 +945,15 @@ var setupTests = []setupTest{{ Version: "22.04", Suites: []string{"jammy-updates"}, Components: []string{"universe"}, + Priority: 10, PubKeys: []*packet.PublicKey{testKey.PubKey, extraTestKey.PubKey}, }, }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "foo", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, @@ -944,7 +967,6 @@ var setupTests = []setupTest{{ version: 22.04 components: [main, universe] suites: [jammy] - default: true `, }, relerror: `chisel.yaml: archive "foo" missing v1-public-keys field`, @@ -959,7 +981,6 @@ var setupTests = []setupTest{{ components: [main, universe] suites: [jammy] v1-public-keys: [extra-key] - default: true `, "slices/mydir/mypkg.yaml": ` package: mypkg @@ -977,7 +998,6 @@ var setupTests = []setupTest{{ components: [main, universe] suites: [jammy] v1-public-keys: [extra-key] - default: true v1-public-keys: extra-key: id: foo @@ -1005,7 +1025,6 @@ var setupTests = []setupTest{{ components: [main, universe] suites: [jammy] v1-public-keys: [extra-key] - default: true v1-public-keys: extra-key: id: ` + extraTestKey.ID + ` @@ -1028,8 +1047,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -1041,9 +1058,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "jq": { - Archive: "ubuntu", - Name: "jq", - Path: "slices/mydir/jq.yaml", + Name: "jq", + Path: "slices/mydir/jq.yaml", Slices: map[string]*setup.Slice{ "bins": { Package: "jq", @@ -1082,7 +1098,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -1094,9 +1109,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1151,8 +1165,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -1164,9 +1176,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "mypkg", @@ -1187,9 +1198,8 @@ var setupTests = []setupTest{{ }, }, "myotherpkg": { - Archive: "ubuntu", - Name: "myotherpkg", - Path: "slices/mydir/myotherpkg.yaml", + Name: "myotherpkg", + Path: "slices/mydir/myotherpkg.yaml", Slices: map[string]*setup.Slice{ "slice1": { Package: "myotherpkg", @@ -1311,6 +1321,15 @@ var setupTests = []setupTest{{ `, }, relerror: `slices test-package_myslice1 and test-package_myslice2 conflict on /dir/\*\* and /dir/file`, +}, { + summary: "Pinned archive is not defined", + input: map[string]string{ + "slices/test-package.yaml": ` + package: test-package + archive: non-existing + `, + }, + relerror: `slices/test-package.yaml: package refers to undefined archive "non-existing"`, }, { summary: "Specify generate: manifest", input: map[string]string{ @@ -1323,8 +1342,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -1336,9 +1353,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1373,8 +1389,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -1386,9 +1400,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1465,8 +1478,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -1478,9 +1489,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -1492,9 +1502,8 @@ var setupTests = []setupTest{{ }, }, "mypkg2": { - Archive: "ubuntu", - Name: "mypkg2", - Path: "slices/mydir/mypkg2.yaml", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg2", @@ -1562,6 +1571,45 @@ var setupTests = []setupTest{{ `, }, relerror: `slice mypkg_myslice path /path/\*\* has invalid generate options`, +}, { + summary: "Default archive is deprecated and ignored", + input: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + ubuntu: + default: true + version: 22.04 + components: [main] + suites: [jammy] + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, }} var defaultChiselYaml = ` diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index e76b07af..89d8b377 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -86,21 +86,16 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } + archives, err := selectPkgArchives(options.Archives, options.Selection) + if err != nil { + return err + } + // Build information to process the selection. extract := make(map[string]map[string][]deb.ExtractInfo) - archives := make(map[string]archive.Archive) for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { - archiveName := options.Selection.Release.Packages[slice.Package].Archive - archive := options.Archives[archiveName] - if archive == nil { - return fmt.Errorf("archive %q not defined", archiveName) - } - if !archive.Exists(slice.Package) { - return fmt.Errorf("slice package %q missing from archive", slice.Package) - } - archives[slice.Package] = archive extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } @@ -469,3 +464,52 @@ func createFile(targetPath string, pathInfo setup.PathInfo) (*fsutil.Entry, erro MakeParents: true, }) } + +// selectPkgArchives selects the highest priority archive containing the package +// unless a particular archive is pinned within the slice definition file. It +// returns a map of archives indexed by package names. +func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { + sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) + for _, archive := range selection.Release.Archives { + if archive.Priority < 0 { + // Ignore negative priority archives unless a package specifically + // asks for it with the "archive" field. + continue + } + sortedArchives = append(sortedArchives, archive) + } + slices.SortFunc(sortedArchives, func(a, b *setup.Archive) int { + return b.Priority - a.Priority + }) + + pkgArchives := make(map[string]archive.Archive) + for _, s := range selection.Slices { + if _, ok := pkgArchives[s.Package]; ok { + continue + } + pkg := selection.Release.Packages[s.Package] + + var candidates []*setup.Archive + if pkg.Archive == "" { + // If the package has not pinned any archive, choose the highest + // priority archive in which the package exists. + candidates = sortedArchives + } else { + candidates = []*setup.Archive{selection.Release.Archives[pkg.Archive]} + } + + var chosen archive.Archive + for _, archiveInfo := range candidates { + archive := archives[archiveInfo.Name] + if archive != nil && archive.Exists(pkg.Name) { + chosen = archive + break + } + } + if chosen == nil { + return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) + } + pkgArchives[pkg.Name] = chosen + } + return pkgArchives, nil +} diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 8035ef0d..2cbfe1f3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -7,6 +7,7 @@ import ( "os" "path" "path/filepath" + "slices" "sort" "strings" @@ -28,7 +29,7 @@ type slicerTest struct { summary string arch string release map[string]string - pkgs map[string]testutil.TestPackage + pkgs []*testutil.TestPackage slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) filesystem map[string]string @@ -247,12 +248,11 @@ var slicerTests = []slicerTest{{ }, { summary: "Copyright is installed", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string]testutil.TestPackage{ - "test-package": { - // Add the copyright entries to the package. - Data: testutil.MustMakeDeb(append(testutil.TestPackageEntries, testPackageCopyrightEntries...)), - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + // Add the copyright entries to the package. + Data: testutil.MustMakeDeb(append(testutil.TestPackageEntries, testPackageCopyrightEntries...)), + }}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -280,14 +280,13 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, {"other-package", "myslice"}}, - pkgs: map[string]testutil.TestPackage{ - "test-package": { - Data: testutil.PackageData["test-package"], - }, - "other-package": { - Data: testutil.PackageData["other-package"], - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }, { + Name: "other-package", + Data: testutil.PackageData["other-package"], + }}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -324,19 +323,18 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"implicit-parent", "myslice"}, {"explicit-dir", "myslice"}}, - pkgs: map[string]testutil.TestPackage{ - "implicit-parent": { - Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(0755, "./dir/"), - testutil.Reg(0644, "./dir/file", "random"), - }), - }, - "explicit-dir": { - Data: testutil.MustMakeDeb([]testutil.TarEntry{ - testutil.Dir(01777, "./dir/"), - }), - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "implicit-parent", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(0755, "./dir/"), + testutil.Reg(0644, "./dir/file", "random"), + }), + }, { + Name: "explicit-dir", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Dir(01777, "./dir/"), + }), + }}, release: map[string]string{ "slices/mydir/implicit-parent.yaml": ` package: implicit-parent @@ -366,14 +364,13 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, {"other-package", "myslice"}}, - pkgs: map[string]testutil.TestPackage{ - "test-package": { - Data: testutil.PackageData["test-package"], - }, - "other-package": { - Data: testutil.PackageData["other-package"], - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }, { + Name: "other-package", + Data: testutil.PackageData["other-package"], + }}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -704,14 +701,13 @@ var slicerTests = []slicerTest{{ }, { summary: "Duplicate copyright symlink is ignored", slices: []setup.SliceKey{{"copyright-symlink-openssl", "bins"}}, - pkgs: map[string]testutil.TestPackage{ - "copyright-symlink-openssl": { - Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), - }, - "copyright-symlink-libssl3": { - Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-libssl3"]), - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "copyright-symlink-openssl", + Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-openssl"]), + }, { + Name: "copyright-symlink-libssl3", + Data: testutil.MustMakeDeb(packageEntries["copyright-symlink-libssl3"]), + }}, release: map[string]string{ "slices/mydir/copyright-symlink-libssl3.yaml": ` package: copyright-symlink-libssl3 @@ -769,8 +765,106 @@ var slicerTests = []slicerTest{{ }, error: `slice test-package_myslice: content is not a file: /x/y`, }, { - summary: "Non-default archive", + summary: "Multiple archives with priority", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"other-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }, { + Name: "test-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from bar"), + }), + Archives: []string{"bar"}, + }, { + Name: "other-package", + Hash: "h3", + Version: "v3", + Arch: "a3", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./other-file", "from bar"), + }), + Archives: []string{"bar"}, + }}, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [main] + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + "slices/mydir/other-package.yaml": ` + package: other-package + slices: + myslice: + contents: + /other-file: + `, + }, + filesystem: map[string]string{ + // The notion of "default" is obsolete and highest priority is selected. + "/file": "file 0644 7a3e00f5", + // Fetched from archive "bar" as no other archive has the package. + "/other-file": "file 0644 fa0c9cdb", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + "/other-file": "file 0644 fa0c9cdb {other-package_myslice}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 h1", + "other-package": "other-package v3 a3 h3", + }, +}, { + summary: "Pinned archive bypasses higher priority", slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }, { + Name: "test-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from bar"), + }), + Archives: []string{"bar"}, + }}, release: map[string]string{ "chisel.yaml": ` format: chisel-v1 @@ -778,11 +872,12 @@ var slicerTests = []slicerTest{{ foo: version: 22.04 components: [main, universe] - default: true + priority: 20 v1-public-keys: [test-key] bar: version: 22.04 components: [main] + priority: 10 v1-public-keys: [test-key] v1-public-keys: test-key: @@ -795,19 +890,172 @@ var slicerTests = []slicerTest{{ slices: myslice: contents: - /dir/nested/file: + /file: `, }, hackopt: func(c *C, opts *slicer.RunOptions) { delete(opts.Archives, "foo") }, filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/nested/": "dir 0755", - "/dir/nested/file": "file 0644 84237a05", + // test-package fetched from pinned archive "bar". + "/file": "file 0644 fa0c9cdb", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 fa0c9cdb {test-package_myslice}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v2 a2 h2", + }, +}, { + summary: "Pinned archive does not have the package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }}, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [main] + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + archive: bar + slices: + myslice: + contents: + /file: + `, + }, + // Although archive "foo" does have the package, since archive "bar" has + // been pinned in the slice definition, no other archives will be checked. + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "No archives have the package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{}, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + priority: 20 + v1-public-keys: [test-key] + bar: + version: 22.04 + components: [main] + priority: 10 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "Negative priority archives are ignored when not explicitly pinned in package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }}, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + priority: -20 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "Negative priority archive explicitly pinned in package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }}, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + foo: + version: 22.04 + components: [main, universe] + priority: -20 + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + archive: foo + slices: + myslice: + contents: + /file: + `, + }, + filesystem: map[string]string{ + "/file": "file 0644 7a3e00f5", }, manifestPaths: map[string]string{ - "/dir/nested/file": "file 0644 84237a05 {test-package_myslice}", + "/file": "file 0644 7a3e00f5 {test-package_myslice}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 h1", }, }, { summary: "Multiple slices of same package", @@ -1068,22 +1316,19 @@ var slicerTests = []slicerTest{{ {"test-package", "myslice"}, {"other-package", "myslice"}, }, - pkgs: map[string]testutil.TestPackage{ - "test-package": { - Name: "test-package", - Hash: "h1", - Version: "v1", - Arch: "a1", - Data: testutil.PackageData["test-package"], - }, - "other-package": { - Name: "other-package", - Hash: "h2", - Version: "v2", - Arch: "a2", - Data: testutil.PackageData["other-package"], - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.PackageData["test-package"], + }, { + Name: "other-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.PackageData["other-package"], + }}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -1107,22 +1352,19 @@ var slicerTests = []slicerTest{{ slices: []setup.SliceKey{ {"test-package", "myslice"}, }, - pkgs: map[string]testutil.TestPackage{ - "test-package": { - Name: "test-package", - Hash: "h1", - Version: "v1", - Arch: "a1", - Data: testutil.PackageData["test-package"], - }, - "other-package": { - Name: "other-package", - Hash: "h2", - Version: "v2", - Arch: "a2", - Data: testutil.PackageData["other-package"], - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.PackageData["test-package"], + }, { + Name: "other-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.PackageData["other-package"], + }}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -1143,19 +1385,18 @@ var slicerTests = []slicerTest{{ }, { summary: "Relative paths are properly trimmed during extraction", slices: []setup.SliceKey{{"test-package", "myslice"}}, - pkgs: map[string]testutil.TestPackage{ - "test-package": { - Data: testutil.MustMakeDeb([]testutil.TarEntry{ - // This particular path starting with "/foo" is chosen to test for - // a particular bug; which appeared due to the usage of - // strings.TrimLeft() instead strings.TrimPrefix() to determine a - // relative path. Since TrimLeft takes in a cutset instead of a - // prefix, the desired relative path was not produced. - // See https://github.com/canonical/chisel/pull/145. - testutil.Dir(0755, "./foo-bar/"), - }), - }, - }, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + // This particular path starting with "/foo" is chosen to test for + // a particular bug; which appeared due to the usage of + // strings.TrimLeft() instead strings.TrimPrefix() to determine a + // relative path. Since TrimLeft takes in a cutset instead of a + // prefix, the desired relative path was not produced. + // See https://github.com/canonical/chisel/pull/145. + testutil.Dir(0755, "./foo-bar/"), + }), + }}, hackopt: func(c *C, opts *slicer.RunOptions) { opts.TargetDir = filepath.Join(filepath.Clean(opts.TargetDir), "foo") err := os.Mkdir(opts.TargetDir, 0755) @@ -1232,25 +1473,17 @@ func (s *S) TestRun(c *C) { func runSlicerTests(c *C, tests []slicerTest) { for _, test := range tests { - for _, slices := range testutil.Permutations(test.slices) { + for _, testSlices := range testutil.Permutations(test.slices) { c.Logf("Summary: %s", test.summary) if _, ok := test.release["chisel.yaml"]; !ok { test.release["chisel.yaml"] = defaultChiselYaml } if test.pkgs == nil { - test.pkgs = map[string]testutil.TestPackage{ - "test-package": { - Data: testutil.PackageData["test-package"], - }, - } - } - for pkgName, pkg := range test.pkgs { - if pkg.Name == "" { - // We need to add the name for the manifest validation. - pkg.Name = pkgName - test.pkgs[pkgName] = pkg - } + test.pkgs = []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }} } releaseDir := c.MkDir() @@ -1280,16 +1513,22 @@ func runSlicerTests(c *C, tests []slicerTest) { }, Scripts: setup.SliceScripts{}, } - slices = append(slices, setup.SliceKey{ + testSlices = append(testSlices, setup.SliceKey{ Package: manifestPackage, Slice: "manifest", }) - selection, err := setup.Select(release, slices) + selection, err := setup.Select(release, testSlices) c.Assert(err, IsNil) archives := map[string]archive.Archive{} for name, setupArchive := range release.Archives { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { + pkgs[pkg.Name] = pkg + } + } archive := &testutil.TestArchive{ Opts: archive.Options{ Label: setupArchive.Name, @@ -1298,7 +1537,7 @@ func runSlicerTests(c *C, tests []slicerTest) { Components: setupArchive.Components, Arch: test.arch, }, - Packages: test.pkgs, + Packages: pkgs, } archives[name] = archive } diff --git a/internal/testutil/archive.go b/internal/testutil/archive.go index 77b945cc..ae8258c2 100644 --- a/internal/testutil/archive.go +++ b/internal/testutil/archive.go @@ -10,15 +10,16 @@ import ( type TestArchive struct { Opts archive.Options - Packages map[string]TestPackage + Packages map[string]*TestPackage } type TestPackage struct { - Name string - Version string - Hash string - Arch string - Data []byte + Name string + Version string + Hash string + Arch string + Data []byte + Archives []string } func (a *TestArchive) Options() *archive.Options { From 3af0bf1822f0dfc4836c9865a79b36dd0949189e Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 9 Oct 2024 20:26:44 +0600 Subject: [PATCH 19/60] refactor(archive): minor changes, add (real-archive) tests --- internal/archive/archive.go | 13 +++-- internal/archive/archive_test.go | 83 +++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index d49714e7..8575c4b9 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net/http" - "slices" "strings" "time" @@ -174,7 +173,7 @@ var proArchiveInfo = map[string]struct { } // archiveURL returns the archive base URL depending on the "pro" value and -// selected architecture "arch". The "pro" value should be validated. +// selected architecture "arch". The "pro" value must be pre-validated. func archiveURL(pro, arch string) string { if pro != "" { return proArchiveInfo[pro].BaseURL @@ -234,7 +233,8 @@ func openUbuntu(options *Options) (Archive, error) { if !index.supportsArch(options.Arch) { // Release does not support the specified architecture. Do // not add indexes of this release. - logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...", index.proSuffixedLabel(), index.version, index.suite, options.Arch) + logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...", + index.proSuffixedLabel(), index.version, index.suite, options.Arch) break } err = index.checkComponents(options.Components) @@ -325,9 +325,12 @@ func (index *ubuntuIndex) fetchIndex() error { return nil } +// supportsArch returns true if the Architectures field in the index release +// contains "arch". Per the Debian wiki [1], index release files should list the +// supported architectures in the "Architectures" field. +// Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures func (index *ubuntuIndex) supportsArch(arch string) bool { - supportedArches := strings.Fields(index.release.Get("Architectures")) - return slices.Contains(supportedArches, arch) + return strings.Contains(index.release.Get("Architectures"), arch) } func (index *ubuntuIndex) checkComponents(components []string) error { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 27ad4e14..663cbc11 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -396,6 +396,15 @@ func (s *httpSuite) TestProArchives(c *C) { err := os.WriteFile(confFile, []byte(contents), 0600) c.Assert(err, IsNil) + do := func(req *http.Request) (*http.Response, error) { + auth, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + c.Assert(auth, DeepEquals, []string{"Basic Zm9vOmJhcg=="}) + return s.Do(req) + } + restoreDo := archive.FakeDo(do) + defer restoreDo() + for pro, info := range archive.ProArchiveInfo { s.base = info.BaseURL s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel(info.Label)) @@ -416,6 +425,32 @@ func (s *httpSuite) TestProArchives(c *C) { } } +func (s *httpSuite) TestNonProArchive(c *C) { + do := func(req *http.Request) (*http.Response, error) { + if _, ok := req.Header["Authorization"]; ok { + c.Fatalf("Non-pro archives should not have any authorization header") + } + return s.Do(req) + } + restoreDo := archive.FakeDo(do) + defer restoreDo() + + s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + _, err := archive.Open(&options) + c.Assert(err, IsNil) +} + type verifyArchiveReleaseTest struct { summary string pubKeys []*packet.PublicKey @@ -518,30 +553,39 @@ func read(r io.Reader) string { } // ---------------------------------------------------------------------------------------- -// Real archive tests, only enabled via --real-archive. +// Real archive tests, only enabled via: +// 1. --real-archive for non-Pro archives (e.g. standard jammy archive) +// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). // -// Additionally, run tests against Ubuntu Pro archives by passing --pro-archive. -// The host machine must be Pro enabled and relevant Pro services must be -// enabled. The following commands might help: +// To run the tests for Ubuntu Pro archives, the host machine must be Pro +// enabled and relevant Pro services must be enabled. The following commands +// might help: // sudo pro attach --no-auto-enable -// sudo pro enable fips fips-updates esm-apps esm-infra --assume-yes +// sudo pro enable fips-updates esm-apps esm-infra --assume-yes var realArchiveFlag = flag.Bool("real-archive", false, "Perform tests against real archive") -var proArchiveFlag = flag.Bool("pro-archive", false, "Perform tests against real Ubuntu Pro archive") +var proArchiveFlag = flag.Bool("real-pro-archive", false, "Perform tests against real Ubuntu Pro archive") func (s *S) TestRealArchive(c *C) { if !*realArchiveFlag { c.Skip("--real-archive not provided") } + s.runRealArchiveTests(c, realArchiveTests) +} + +func (s *S) TestProArchives(c *C) { + if !*proArchiveFlag { + c.Skip("--real-pro-archive not provided") + } + s.runRealArchiveTests(c, proArchiveTests) +} + +func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { allArch := make([]string, 0, len(elfToDebArch)) for _, arch := range elfToDebArch { allArch = append(allArch, arch) } - for _, test := range realArchiveTests { - if test.pro != "" && !*proArchiveFlag { - // --pro-archive is not provided. Ignore test. - continue - } + for _, test := range tests { if len(test.architectures) == 0 { test.architectures = allArch } @@ -591,18 +635,11 @@ var realArchiveTests = []realArchiveTest{{ pkg: "hostname", binPath: "/usr/bin/hostname", copyrightText: "This package was written by Peter Tobias ", -}, { - name: "fips", - version: "20.04", - suites: []string{"focal"}, - components: []string{"main"}, - pro: "fips", - archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, - architectures: []string{"amd64"}, - pkg: "openssh-client", - binPath: "/usr/bin/ssh", - copyrightText: "1995 Tatu Ylonen , Espoo, Finland", -}, { +}} + +var proArchiveTests = []realArchiveTest{{ + // We cannot test both fips and fips-updates since both services cannot be + // enabled at the same time. name: "fips-updates", version: "20.04", suites: []string{"focal-updates"}, From 7b7bc28fb8140ef4734f71d7c6bfb91248cf9ca2 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 9 Oct 2024 20:27:44 +0600 Subject: [PATCH 20/60] refactor: move "empty archives list" check to slicer --- internal/setup/setup.go | 3 --- internal/setup/setup_test.go | 16 ---------------- internal/slicer/slicer.go | 3 +++ internal/slicer/slicer_test.go | 23 +++++++++++++++++++++++ 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index d91c8048..2922542f 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -597,9 +597,6 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { PubKeys: archiveKeys, } } - if len(release.Archives) == 0 { - return nil, fmt.Errorf("%s: no valid archives defined", fileName) - } return release, err } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 6d89975f..84c7194b 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1739,22 +1739,6 @@ var setupTests = []setupTest{{ }, }, }, -}, { - summary: "No valid archives defined", - input: map[string]string{ - "chisel.yaml": ` - format: v1 - archives: - invalid: - version: 20.04 - components: [main] - suites: [focal] - priority: 10 - public-keys: [test-key] - pro: unknown-value - `, - }, - relerror: `chisel.yaml: no valid archives defined`, }} var defaultChiselYaml = ` diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 89d8b377..31683704 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -86,6 +86,9 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } + if len(options.Archives) == 0 { + return fmt.Errorf("no valid archives defined") + } archives, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 32333c61..98c62fa8 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1435,6 +1435,29 @@ var slicerTests = []slicerTest{{ contents: `, }, +}, { + summary: "No valid archives defined due to invalid pro value", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + invalid: + version: 20.04 + components: [main] + suites: [focal] + priority: 10 + public-keys: [test-key] + pro: unknown-value + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + `, + }, + error: `no valid archives defined`, }} var defaultChiselYaml = ` From ec1637513f6d87f17a17ec46621c84bb12a6ab86 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 9 Oct 2024 20:40:51 +0600 Subject: [PATCH 21/60] test(archive): use proper pkg in real archive test --- internal/archive/archive_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 663cbc11..46062eae 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -712,9 +712,9 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { extractDir := c.MkDir() - pkg, info, err := testArchive.Fetch("hostname") + pkg, info, err := testArchive.Fetch(test.pkg) c.Assert(err, IsNil) - c.Assert(info.Name, DeepEquals, "hostname") + c.Assert(info.Name, DeepEquals, test.pkg) c.Assert(info.Arch, DeepEquals, arch) err = deb.Extract(pkg, &deb.ExtractOptions{ From a4d9407ba9c9c7b75131266866585da2a6f98dc5 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 9 Oct 2024 20:53:04 +0600 Subject: [PATCH 22/60] test(archive): add more real pro archives test --- internal/archive/archive_test.go | 61 ++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 46062eae..fd84bd95 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -640,7 +640,7 @@ var realArchiveTests = []realArchiveTest{{ var proArchiveTests = []realArchiveTest{{ // We cannot test both fips and fips-updates since both services cannot be // enabled at the same time. - name: "fips-updates", + name: "focal-fips-updates", version: "20.04", suites: []string{"focal-updates"}, components: []string{"main"}, @@ -651,7 +651,7 @@ var proArchiveTests = []realArchiveTest{{ binPath: "/usr/bin/ssh", copyrightText: "1995 Tatu Ylonen , Espoo, Finland", }, { - name: "esm-apps", + name: "focal-esm-apps", version: "20.04", suites: []string{"focal-apps-security", "focal-apps-updates"}, components: []string{"main"}, @@ -662,7 +662,7 @@ var proArchiveTests = []realArchiveTest{{ binPath: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }, { - name: "esm-infra", + name: "focal-esm-infra", version: "20.04", suites: []string{"focal-infra-security", "focal-infra-updates"}, components: []string{"main"}, @@ -672,6 +672,61 @@ var proArchiveTests = []realArchiveTest{{ pkg: "hello", binPath: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", +}, { + name: "jammy-fips-updates", + version: "22.04", + suites: []string{"jammy-updates"}, + components: []string{"main"}, + pro: "fips-updates", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + architectures: []string{"amd64"}, + pkg: "openssh-client", + binPath: "/usr/bin/ssh", + copyrightText: "1995 Tatu Ylonen , Espoo, Finland", +}, { + name: "jammy-esm-apps", + version: "22.04", + suites: []string{"jammy-apps-security", "jammy-apps-updates"}, + components: []string{"main"}, + pro: "apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + architectures: []string{"amd64"}, + pkg: "hello", + binPath: "/usr/bin/hello", + copyrightText: "This package was first put together by Ian Jackson", +}, { + name: "jammy-esm-infra", + version: "22.04", + suites: []string{"jammy-infra-security", "jammy-infra-updates"}, + components: []string{"main"}, + pro: "infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + architectures: []string{"amd64"}, + pkg: "hello", + binPath: "/usr/bin/hello", + copyrightText: "This package was first put together by Ian Jackson", +}, { + name: "noble-esm-apps", + version: "24.04", + suites: []string{"noble-apps-security", "noble-apps-updates"}, + components: []string{"main"}, + pro: "apps", + archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, + architectures: []string{"amd64"}, + pkg: "hello", + binPath: "/usr/bin/hello", + copyrightText: "This package was first put together by Ian Jackson", +}, { + name: "noble-esm-infra", + version: "24.04", + suites: []string{"noble-infra-security", "noble-infra-updates"}, + components: []string{"main"}, + pro: "infra", + archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, + architectures: []string{"amd64"}, + pkg: "hello", + binPath: "/usr/bin/hello", + copyrightText: "This package was first put together by Ian Jackson", }} var elfToDebArch = map[elf.Machine]string{ From 49b84a004923d5ea5754b29f8b153fde6c328d04 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 9 Oct 2024 21:54:11 +0600 Subject: [PATCH 23/60] ci: add workflow job to run pro archive tests --- .github/workflows/tests.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7579f22d..312e3b94 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -43,3 +43,32 @@ jobs: with: name: chisel-test-coverage.html path: ./*.html + + real-archive-tests: + # Do not change to ubuntu-latest as fips-updates may not be available there. + runs-on: ubuntu-22.04 + name: Real Archive Tests + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + - name: Run real archive tests + env: + PRO_TOKEN: ${{ secrets.PRO_TOKEN }} + run: | + set -ex + + detach() { + sudo pro detach --assume-yes || true + } + trap detach EXIT + + # Attach pro token and enable services + sudo pro attach ${PRO_TOKEN} --no-auto-enable + sudo pro enable fips-updates esm-apps esm-infra --assume-yes + + # Run tests on Pro and non-Pro real archives. + go test ./internal/archive/ -v --real-archive --real-pro-archive From 80ea7b22449ae2e75938e0b8ba950c2644b05c74 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 9 Oct 2024 22:13:28 +0600 Subject: [PATCH 24/60] ci: fix permissions to run the real archive tests --- .github/workflows/tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 312e3b94..6b4dbc62 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -63,6 +63,7 @@ jobs: detach() { sudo pro detach --assume-yes || true + sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage } trap detach EXIT @@ -70,5 +71,8 @@ jobs: sudo pro attach ${PRO_TOKEN} --no-auto-enable sudo pro enable fips-updates esm-apps esm-infra --assume-yes + # Make apt credentials accessible to USER. + sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage + # Run tests on Pro and non-Pro real archives. go test ./internal/archive/ -v --real-archive --real-pro-archive From a19c876a48e3fdcfb204886fdfd11173a25055c1 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Mon, 14 Oct 2024 16:51:16 +0200 Subject: [PATCH 25/60] more descriptive variable name --- internal/slicer/slicer.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 89d8b377..b45d2065 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -86,7 +86,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - archives, err := selectPkgArchives(options.Archives, options.Selection) + pkgToArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err } @@ -99,7 +99,7 @@ func Run(options *RunOptions) error { extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } - arch := archives[slice.Package].Options().Arch + arch := pkgToArchive[slice.Package].Options().Arch copyrightPath := "/usr/share/doc/" + slice.Package + "/copyright" hasCopyright := false for targetPath, pathInfo := range slice.Contents { @@ -151,7 +151,7 @@ func Run(options *RunOptions) error { if packages[slice.Package] != nil { continue } - reader, info, err := archives[slice.Package].Fetch(slice.Package) + reader, info, err := pkgToArchive[slice.Package].Fetch(slice.Package) if err != nil { return err } @@ -248,7 +248,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := archives[slice.Package].Options().Arch + arch := pkgToArchive[slice.Package].Options().Arch for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -482,9 +482,9 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return b.Priority - a.Priority }) - pkgArchives := make(map[string]archive.Archive) + pkgToArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - if _, ok := pkgArchives[s.Package]; ok { + if _, ok := pkgToArchive[s.Package]; ok { continue } pkg := selection.Release.Packages[s.Package] @@ -509,7 +509,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } - pkgArchives[pkg.Name] = chosen + pkgToArchive[pkg.Name] = chosen } - return pkgArchives, nil + return pkgToArchive, nil } From bb18527ae836fe79bdc978cbd8f2eb5993e9dcd9 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Mon, 14 Oct 2024 19:25:17 +0200 Subject: [PATCH 26/60] pkgToArchive -> pkgArchive --- internal/slicer/slicer.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index b45d2065..2991e253 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -86,7 +86,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgToArchive, err := selectPkgArchives(options.Archives, options.Selection) + pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err } @@ -99,7 +99,7 @@ func Run(options *RunOptions) error { extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } - arch := pkgToArchive[slice.Package].Options().Arch + arch := pkgArchive[slice.Package].Options().Arch copyrightPath := "/usr/share/doc/" + slice.Package + "/copyright" hasCopyright := false for targetPath, pathInfo := range slice.Contents { @@ -151,7 +151,7 @@ func Run(options *RunOptions) error { if packages[slice.Package] != nil { continue } - reader, info, err := pkgToArchive[slice.Package].Fetch(slice.Package) + reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) if err != nil { return err } @@ -248,7 +248,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgToArchive[slice.Package].Options().Arch + arch := pkgArchive[slice.Package].Options().Arch for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -482,9 +482,9 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return b.Priority - a.Priority }) - pkgToArchive := make(map[string]archive.Archive) + pkgArchive := make(map[string]archive.Archive) for _, s := range selection.Slices { - if _, ok := pkgToArchive[s.Package]; ok { + if _, ok := pkgArchive[s.Package]; ok { continue } pkg := selection.Release.Packages[s.Package] @@ -509,7 +509,7 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel if chosen == nil { return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.Name) } - pkgToArchive[pkg.Name] = chosen + pkgArchive[pkg.Name] = chosen } - return pkgToArchive, nil + return pkgArchive, nil } From e634cffe1a884a25cd59dd32a73621d11829c759 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:21:15 +0600 Subject: [PATCH 27/60] doc: add proper comments about arch support of ports --- internal/archive/archive.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 8575c4b9..93c0dc26 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -328,6 +328,9 @@ func (index *ubuntuIndex) fetchIndex() error { // supportsArch returns true if the Architectures field in the index release // contains "arch". Per the Debian wiki [1], index release files should list the // supported architectures in the "Architectures" field. +// The "ubuntuURL" archive only supports the amd64 and i386 architectures +// whereas the "ubuntuPortsURL" one supports the rest. But each of them +// (faultly) specifies all those architectures in their InRelease files. // Reference: [1] https://wiki.debian.org/DebianRepository/Format#Architectures func (index *ubuntuIndex) supportsArch(arch string) bool { return strings.Contains(index.release.Get("Architectures"), arch) From 374acb25f2dc7e221a2db98cc8ea79e71ccdaf6a Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:30:24 +0600 Subject: [PATCH 28/60] test: user gocheck idiomatically https://github.com/letFunny/chisel/pull/13#discussion_r1793869205 --- internal/archive/archive_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index fd84bd95..bd5ddb70 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -427,9 +427,8 @@ func (s *httpSuite) TestProArchives(c *C) { func (s *httpSuite) TestNonProArchive(c *C) { do := func(req *http.Request) (*http.Response, error) { - if _, ok := req.Header["Authorization"]; ok { - c.Fatalf("Non-pro archives should not have any authorization header") - } + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, false, Commentf("Non-pro archives should not have any authorization header")) return s.Do(req) } restoreDo := archive.FakeDo(do) From 3289ac7e80cd062ee75f2e9b67fd0ce6224215ff Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:37:20 +0600 Subject: [PATCH 29/60] doc: nitpick: use spaces instead of tabs in comments --- internal/archive/archive_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index bd5ddb70..3e664194 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -553,14 +553,14 @@ func read(r io.Reader) string { // ---------------------------------------------------------------------------------------- // Real archive tests, only enabled via: -// 1. --real-archive for non-Pro archives (e.g. standard jammy archive) -// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). +// 1. --real-archive for non-Pro archives (e.g. standard jammy archive), +// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). // // To run the tests for Ubuntu Pro archives, the host machine must be Pro // enabled and relevant Pro services must be enabled. The following commands // might help: -// sudo pro attach --no-auto-enable -// sudo pro enable fips-updates esm-apps esm-infra --assume-yes +// sudo pro attach --no-auto-enable +// sudo pro enable fips-updates esm-apps esm-infra --assume-yes var realArchiveFlag = flag.Bool("real-archive", false, "Perform tests against real archive") var proArchiveFlag = flag.Bool("real-pro-archive", false, "Perform tests against real Ubuntu Pro archive") From 607d097142b24a1bcb58bdd405c0f6840142815a Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:41:42 +0600 Subject: [PATCH 30/60] test(archive): s/binPath/path --- internal/archive/archive_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 3e664194..79d2496b 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -603,7 +603,7 @@ type realArchiveTest struct { archivePubKeys []*packet.PublicKey architectures []string pkg string - binPath string + path string copyrightText string } @@ -614,7 +614,7 @@ var realArchiveTests = []realArchiveTest{{ components: []string{"main", "universe"}, archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, pkg: "hostname", - binPath: "/bin/hostname", + path: "/bin/hostname", copyrightText: "This package was written by Peter Tobias ", }, { name: "jammy", @@ -623,7 +623,7 @@ var realArchiveTests = []realArchiveTest{{ components: []string{"main", "universe"}, archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, pkg: "hostname", - binPath: "/bin/hostname", + path: "/bin/hostname", copyrightText: "This package was written by Peter Tobias ", }, { name: "noble", @@ -632,7 +632,7 @@ var realArchiveTests = []realArchiveTest{{ components: []string{"main", "universe"}, archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, pkg: "hostname", - binPath: "/usr/bin/hostname", + path: "/usr/bin/hostname", copyrightText: "This package was written by Peter Tobias ", }} @@ -647,7 +647,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, architectures: []string{"amd64"}, pkg: "openssh-client", - binPath: "/usr/bin/ssh", + path: "/usr/bin/ssh", copyrightText: "1995 Tatu Ylonen , Espoo, Finland", }, { name: "focal-esm-apps", @@ -658,7 +658,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, architectures: []string{"amd64"}, pkg: "hello", - binPath: "/usr/bin/hello", + path: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }, { name: "focal-esm-infra", @@ -669,7 +669,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, architectures: []string{"amd64"}, pkg: "hello", - binPath: "/usr/bin/hello", + path: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }, { name: "jammy-fips-updates", @@ -680,7 +680,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, architectures: []string{"amd64"}, pkg: "openssh-client", - binPath: "/usr/bin/ssh", + path: "/usr/bin/ssh", copyrightText: "1995 Tatu Ylonen , Espoo, Finland", }, { name: "jammy-esm-apps", @@ -691,7 +691,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, architectures: []string{"amd64"}, pkg: "hello", - binPath: "/usr/bin/hello", + path: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }, { name: "jammy-esm-infra", @@ -702,7 +702,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, architectures: []string{"amd64"}, pkg: "hello", - binPath: "/usr/bin/hello", + path: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }, { name: "noble-esm-apps", @@ -713,7 +713,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, architectures: []string{"amd64"}, pkg: "hello", - binPath: "/usr/bin/hello", + path: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }, { name: "noble-esm-infra", @@ -724,7 +724,7 @@ var proArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, architectures: []string{"amd64"}, pkg: "hello", - binPath: "/usr/bin/hello", + path: "/usr/bin/hello", copyrightText: "This package was first put together by Ian Jackson", }} @@ -778,7 +778,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): { {Path: "/copyright"}, }, - test.binPath: { + test.path: { {Path: "/binary"}, }, }, From 6cb7775f59661b66c3468ae9c7c6a03408d1c157 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:48:08 +0600 Subject: [PATCH 31/60] fix: use quotes to log invalid pro value --- internal/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 2922542f..d53d3f80 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -570,7 +570,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { switch details.Pro { case "", ProApps, ProFIPS, ProFIPSUpdates, ProInfra: default: - logf("Ignoring archive %q (invalid pro value: %s)...", archiveName, details.Pro) + logf("Ignoring archive %q (invalid pro value: %q)...", archiveName, details.Pro) continue } if len(details.PubKeys) == 0 { From 69ebee4c97b5bc9332cae5703de2cf954428a160 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:49:42 +0600 Subject: [PATCH 32/60] test(setup): edit test summary --- internal/setup/setup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 84c7194b..022ce128 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1629,7 +1629,7 @@ var setupTests = []setupTest{{ }, }, }, { - summary: "Valid Pro values in archives", + summary: "Pro values in archives", input: map[string]string{ "chisel.yaml": ` format: v1 From 3ba9566c0b4a6eb0d8895e833832f48161381226 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 10:52:05 +0600 Subject: [PATCH 33/60] test(setup): rename ignored archive to "ignored" --- internal/setup/setup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 022ce128..b151fea6 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1668,7 +1668,7 @@ var setupTests = []setupTest{{ pro: infra priority: 15 public-keys: [test-key] - foo: + ignored: version: 20.04 components: [main] suites: [foo] From dd75c9270ef17a15a9fc7f7284870917bae7c2f4 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 19:25:58 +0600 Subject: [PATCH 34/60] refactor: move logic up in archive --- internal/archive/archive.go | 21 ++++++++++++--------- internal/archive/archive_test.go | 10 +++++----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 93c0dc26..1e3ab67a 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -195,21 +195,24 @@ func openUbuntu(options *Options) (Archive, error) { return nil, fmt.Errorf("archive options missing version") } + baseURL := archiveURL(options.Pro, options.Arch) + var creds *credentials + if options.Pro != "" { + var err error + creds, err = findCredentials(baseURL) + if err != nil { + return nil, err + } + } + archive := &ubuntuArchive{ options: *options, cache: &cache.Cache{ Dir: options.CacheDir, }, pubKeys: options.PubKeys, - baseURL: archiveURL(options.Pro, options.Arch), - } - - if options.Pro != "" { - creds, err := findCredentials(archive.baseURL) - if err != nil { - return nil, err - } - archive.creds = creds + baseURL: baseURL, + creds: creds, } for _, suite := range options.Suites { diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 79d2496b..b70abab0 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -423,17 +423,17 @@ func (s *httpSuite) TestProArchives(c *C) { _, err = archive.Open(&options) c.Assert(err, IsNil) } -} -func (s *httpSuite) TestNonProArchive(c *C) { - do := func(req *http.Request) (*http.Response, error) { + // Test non-pro archives. + do = func(req *http.Request) (*http.Response, error) { _, ok := req.Header["Authorization"] c.Assert(ok, Equals, false, Commentf("Non-pro archives should not have any authorization header")) return s.Do(req) } - restoreDo := archive.FakeDo(do) + restoreDo = archive.FakeDo(do) defer restoreDo() + s.base = "http://archive.ubuntu.com/ubuntu/" s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) options := archive.Options{ @@ -446,7 +446,7 @@ func (s *httpSuite) TestNonProArchive(c *C) { PubKeys: []*packet.PublicKey{s.pubKey}, } - _, err := archive.Open(&options) + _, err = archive.Open(&options) c.Assert(err, IsNil) } From a8cdd864714de2deee9d9b47b539f0ba769adc68 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 15 Oct 2024 15:28:07 +0200 Subject: [PATCH 35/60] respect default archive if priorities not being used --- internal/setup/setup.go | 27 ++++++++ internal/setup/setup_test.go | 118 +++++++++++++++++++++++++++++++++-- 2 files changed, 141 insertions(+), 4 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 39fea30f..7190c9d0 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -399,6 +399,7 @@ type yamlArchive struct { Suites []string `yaml:"suites"` Components []string `yaml:"components"` Priority int `yaml:"priority"` + Default bool `yaml:"default"` PubKeys []string `yaml:"public-keys"` } @@ -537,6 +538,11 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { pubKeys[keyName] = key } + // For compatibility if there is a default archive set and priorities are + // not being used, we will revert back to the default archive behaviour. + hasDefault := false + hasPriority := false + var defaultArchive string for archiveName, details := range yamlVar.Archives { if details.Version == "" { return nil, fmt.Errorf("%s: archive %q missing version field", fileName, archiveName) @@ -547,6 +553,13 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if len(details.Components) == 0 { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } + if details.Default && defaultArchive != "" { + return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, defaultArchive, archiveName) + } + if details.Default { + defaultArchive = archiveName + hasDefault = true + } if len(details.PubKeys) == 0 { return nil, fmt.Errorf("%s: archive %q missing public-keys field", fileName, archiveName) } @@ -561,6 +574,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if details.Priority > MaxArchivePriority || details.Priority < MinArchivePriority { return nil, fmt.Errorf("%s: archive %q has invalid priority value of %d", fileName, archiveName, details.Priority) } + if details.Priority != 0 { + hasPriority = true + } release.Archives[archiveName] = &Archive{ Name: archiveName, Version: details.Version, @@ -570,6 +586,17 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { PubKeys: archiveKeys, } } + if hasDefault && !hasPriority { + // For compatibility with the default archive behaviour we will set + // negative priorities to all but the default one, which means all + // others will be ignored unless pinned. + i := -1 + for archiveName, _ := range yamlVar.Archives { + release.Archives[archiveName].Priority = i + i-- + } + release.Archives[defaultArchive].Priority = 0 + } return release, err } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 26105238..79fc9707 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1602,17 +1602,27 @@ var setupTests = []setupTest{{ }, relerror: `chisel.yaml: unknown format "chisel-v1"`, }, { - summary: "Default archive is deprecated and ignored", + summary: "Default archive compatibility", input: map[string]string{ "chisel.yaml": ` format: v1 archives: - ubuntu: + default: default: true version: 22.04 components: [main] suites: [jammy] public-keys: [test-key] + other-1: + version: 22.04 + components: [main] + suites: [jammy] + public-keys: [test-key] + other-2: + version: 22.04 + components: [main] + suites: [jammy] + public-keys: [test-key] public-keys: test-key: id: ` + testKey.ID + ` @@ -1624,12 +1634,29 @@ var setupTests = []setupTest{{ }, release: &setup.Release{ Archives: map[string]*setup.Archive{ - "ubuntu": { - Name: "ubuntu", + "default": { + Name: "default", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Priority: 0, + }, + "other-1": { + Name: "other-1", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Priority: -2, + }, + "other-2": { + Name: "other-2", Version: "22.04", Suites: []string{"jammy"}, Components: []string{"main"}, PubKeys: []*packet.PublicKey{testKey.PubKey}, + Priority: -3, }, }, Packages: map[string]*setup.Package{ @@ -1640,6 +1667,89 @@ var setupTests = []setupTest{{ }, }, }, +}, { + summary: "Default is ignored", + input: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + default: + default: true + priority: 10 + version: 22.04 + components: [main] + suites: [jammy] + public-keys: [test-key] + other: + priority: 20 + version: 22.04 + components: [main] + suites: [jammy] + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + release: &setup.Release{ + Archives: map[string]*setup.Archive{ + "default": { + Name: "default", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Priority: 10, + }, + "other": { + Name: "other", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + Priority: 20, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, + }, + }, + }, +}, { + summary: "Multiple default archives", + input: map[string]string{ + "chisel.yaml": ` + format: v1 + archives: + foo: + default: true + version: 22.04 + components: [main] + suites: [jammy] + public-keys: [test-key] + bar: + default: true + version: 22.04 + components: [main, universe] + suites: [jammy] + v1-public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/mypkg.yaml": ` + package: mypkg + `, + }, + relerror: `chisel.yaml: more than one default archive: foo, bar`, }} var defaultChiselYaml = ` From 40d08675a75bbca18910b7952c756bec8f4c7c83 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 15 Oct 2024 15:39:17 +0200 Subject: [PATCH 36/60] simplify for loop, unnecessary variable --- internal/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 7190c9d0..f62c72d1 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -591,7 +591,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { // negative priorities to all but the default one, which means all // others will be ignored unless pinned. i := -1 - for archiveName, _ := range yamlVar.Archives { + for archiveName := range yamlVar.Archives { release.Archives[archiveName].Priority = i i-- } From 1928bd8b318ed37d9250253ba801326fc28d4626 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 20:26:05 +0600 Subject: [PATCH 37/60] test(archive): no copyright check in real archive tests --- internal/archive/archive_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index b70abab0..8e151d3b 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -604,7 +604,6 @@ type realArchiveTest struct { architectures []string pkg string path string - copyrightText string } var realArchiveTests = []realArchiveTest{{ @@ -615,7 +614,6 @@ var realArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, pkg: "hostname", path: "/bin/hostname", - copyrightText: "This package was written by Peter Tobias ", }, { name: "jammy", version: "22.04", @@ -624,7 +622,6 @@ var realArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, pkg: "hostname", path: "/bin/hostname", - copyrightText: "This package was written by Peter Tobias ", }, { name: "noble", version: "24.04", @@ -633,7 +630,6 @@ var realArchiveTests = []realArchiveTest{{ archivePubKeys: []*packet.PublicKey{keyUbuntu2018.PubKey}, pkg: "hostname", path: "/usr/bin/hostname", - copyrightText: "This package was written by Peter Tobias ", }} var proArchiveTests = []realArchiveTest{{ @@ -648,7 +644,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "openssh-client", path: "/usr/bin/ssh", - copyrightText: "1995 Tatu Ylonen , Espoo, Finland", }, { name: "focal-esm-apps", version: "20.04", @@ -659,7 +654,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", - copyrightText: "This package was first put together by Ian Jackson", }, { name: "focal-esm-infra", version: "20.04", @@ -670,7 +664,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", - copyrightText: "This package was first put together by Ian Jackson", }, { name: "jammy-fips-updates", version: "22.04", @@ -681,7 +674,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "openssh-client", path: "/usr/bin/ssh", - copyrightText: "1995 Tatu Ylonen , Espoo, Finland", }, { name: "jammy-esm-apps", version: "22.04", @@ -692,7 +684,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", - copyrightText: "This package was first put together by Ian Jackson", }, { name: "jammy-esm-infra", version: "22.04", @@ -703,7 +694,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", - copyrightText: "This package was first put together by Ian Jackson", }, { name: "noble-esm-apps", version: "24.04", @@ -714,7 +704,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", - copyrightText: "This package was first put together by Ian Jackson", }, { name: "noble-esm-infra", version: "24.04", @@ -725,7 +714,6 @@ var proArchiveTests = []realArchiveTest{{ architectures: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", - copyrightText: "This package was first put together by Ian Jackson", }} var elfToDebArch = map[elf.Machine]string{ @@ -785,9 +773,5 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { }) c.Assert(err, IsNil) - data, err := os.ReadFile(filepath.Join(extractDir, "copyright")) - c.Assert(err, IsNil) - c.Assert(strings.Contains(string(data), test.copyrightText), Equals, true) - s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) } From 9c22dfd1347a7884c74b961bea0cb28710c35675 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 20:53:07 +0600 Subject: [PATCH 38/60] test(archive): use table tests in TestArchiveLabels --- internal/archive/archive_test.go | 75 +++++++++++++++----------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 8e151d3b..2b593eea 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -331,50 +331,47 @@ func (s *httpSuite) TestArchiveLabels(c *C) { } } - s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + tests := []struct { + summary string + label string + err string + }{{ + summary: "No labels", + }, { + summary: "Ubuntu label", + label: "Ubuntu", + }, { + summary: "Unknown label", + label: "Unknown", + err: "corrupted archive InRelease file: no Ubuntu section", + }} - options := archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, - } - - _, err := archive.Open(&options) - c.Assert(err, IsNil) - - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("Ubuntu")) - - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, - } + for _, test := range tests { + c.Logf("Summary: %s", test.summary) - _, err = archive.Open(&options) - c.Assert(err, IsNil) + var adjust func(*testarchive.Release) + if test.label != "" { + adjust = setLabel(test.label) + } + s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, adjust) - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel("Unknown")) + options := archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + CacheDir: c.MkDir(), + PubKeys: []*packet.PublicKey{s.pubKey}, + } - options = archive.Options{ - Label: "ubuntu", - Version: "22.04", - Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, - CacheDir: c.MkDir(), - PubKeys: []*packet.PublicKey{s.pubKey}, + _, err := archive.Open(&options) + if test.err != "" { + c.Assert(err, ErrorMatches, test.err) + } else { + c.Assert(err, IsNil) + } } - - _, err = archive.Open(&options) - c.Assert(err, ErrorMatches, `corrupted archive InRelease file: no Ubuntu section`) } func (s *httpSuite) TestProArchives(c *C) { From 49e1abfa85ffb4e8c05a16ee7e5400cc4e92bdaa Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 22:11:17 +0600 Subject: [PATCH 39/60] test: add fips real archive test --- .github/workflows/tests.yaml | 11 +++++++++++ internal/archive/archive_test.go | 12 ++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6b4dbc62..c7154b04 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -69,7 +69,18 @@ jobs: # Attach pro token and enable services sudo pro attach ${PRO_TOKEN} --no-auto-enable + + # Cannot enable fips and fips-updates at the same time. + # Hack: enable fips, copy the credentials and then after enabling + # other services, add the credentials back. + sudo pro enable fips --assume-yes + sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds + + # This will disable the fips service. sudo pro enable fips-updates esm-apps esm-infra --assume-yes + # Add the fips credentials back. + sudo cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage + sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds # Make apt credentials accessible to USER. sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 2b593eea..7a9ed495 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -630,8 +630,16 @@ var realArchiveTests = []realArchiveTest{{ }} var proArchiveTests = []realArchiveTest{{ - // We cannot test both fips and fips-updates since both services cannot be - // enabled at the same time. + name: "focal-fips", + version: "20.04", + suites: []string{"focal"}, + components: []string{"main"}, + pro: "fips", + archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + architectures: []string{"amd64"}, + pkg: "openssh-client", + path: "/usr/bin/ssh", +}, { name: "focal-fips-updates", version: "20.04", suites: []string{"focal-updates"}, From fdf812a56bd42bb8809633768db6d1a90198a236 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 22:14:10 +0600 Subject: [PATCH 40/60] chore: use focal to run real archive tests FIPS is not available in Jammy yet. --- .github/workflows/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c7154b04..a2e7e1fc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -45,8 +45,8 @@ jobs: path: ./*.html real-archive-tests: - # Do not change to ubuntu-latest as fips-updates may not be available there. - runs-on: ubuntu-22.04 + # Do not change to newer releases as "fips" may not be available there. + runs-on: ubuntu-20.04 name: Real Archive Tests steps: - uses: actions/checkout@v3 From 5a260b3393d735cfab98312f7b3a159f7878839f Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Tue, 15 Oct 2024 22:23:17 +0600 Subject: [PATCH 41/60] chore: use sudo to write to apt creds file --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a2e7e1fc..4cf514ea 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -79,7 +79,7 @@ jobs: # This will disable the fips service. sudo pro enable fips-updates esm-apps esm-infra --assume-yes # Add the fips credentials back. - sudo cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage + sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage' sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds # Make apt credentials accessible to USER. From 7d1e3f0d75f99481c185eae070b2723ef88d41c5 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Wed, 16 Oct 2024 11:27:29 +0200 Subject: [PATCH 42/60] remove unnecessary bool flag --- internal/setup/setup.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index f62c72d1..3381a6ed 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -540,7 +540,6 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { // For compatibility if there is a default archive set and priorities are // not being used, we will revert back to the default archive behaviour. - hasDefault := false hasPriority := false var defaultArchive string for archiveName, details := range yamlVar.Archives { @@ -558,7 +557,6 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { } if details.Default { defaultArchive = archiveName - hasDefault = true } if len(details.PubKeys) == 0 { return nil, fmt.Errorf("%s: archive %q missing public-keys field", fileName, archiveName) @@ -586,7 +584,7 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { PubKeys: archiveKeys, } } - if hasDefault && !hasPriority { + if defaultArchive != "" && !hasPriority { // For compatibility with the default archive behaviour we will set // negative priorities to all but the default one, which means all // others will be ignored unless pinned. From 118322602beb0d9f64c1b90c05ae43b9046c69bf Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Wed, 16 Oct 2024 11:46:19 +0200 Subject: [PATCH 43/60] make tests deterministic --- internal/setup/setup.go | 10 +++++++--- internal/setup/setup_test.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 3381a6ed..40999bd2 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -588,10 +588,14 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { // For compatibility with the default archive behaviour we will set // negative priorities to all but the default one, which means all // others will be ignored unless pinned. - i := -1 + var archiveNames []string for archiveName := range yamlVar.Archives { - release.Archives[archiveName].Priority = i - i-- + archiveNames = append(archiveNames, archiveName) + } + // Make it deterministic. + slices.Sort(archiveNames) + for i, archiveName := range archiveNames { + release.Archives[archiveName].Priority = -i - 1 } release.Archives[defaultArchive].Priority = 0 } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 79fc9707..43ae6faa 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1739,7 +1739,7 @@ var setupTests = []setupTest{{ version: 22.04 components: [main, universe] suites: [jammy] - v1-public-keys: [test-key] + public-keys: [test-key] public-keys: test-key: id: ` + testKey.ID + ` From bb5f58b516e11577d60ac0b659a6873fd57a79c6 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Wed, 16 Oct 2024 11:48:42 +0200 Subject: [PATCH 44/60] make more tests deterministic --- internal/setup/setup.go | 3 +++ internal/setup/setup_test.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 40999bd2..8f25e5da 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -553,6 +553,9 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } if details.Default && defaultArchive != "" { + if archiveName < defaultArchive { + archiveName, defaultArchive = defaultArchive, archiveName + } return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, defaultArchive, archiveName) } if details.Default { diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 43ae6faa..84704921 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1749,7 +1749,7 @@ var setupTests = []setupTest{{ package: mypkg `, }, - relerror: `chisel.yaml: more than one default archive: foo, bar`, + relerror: `chisel.yaml: more than one default archive: bar, foo`, }} var defaultChiselYaml = ` From 44dcaaa263ab5511c7794108bf05ea6c6a34c690 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 16:27:20 +0600 Subject: [PATCH 45/60] doc: use spaces to align in comments --- internal/archive/archive_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 7a9ed495..df51b3b5 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -550,8 +550,8 @@ func read(r io.Reader) string { // ---------------------------------------------------------------------------------------- // Real archive tests, only enabled via: -// 1. --real-archive for non-Pro archives (e.g. standard jammy archive), -// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). +// 1. --real-archive for non-Pro archives (e.g. standard jammy archive), +// 2. --real-pro-archive for Ubuntu Pro archives (e.g. FIPS archives). // // To run the tests for Ubuntu Pro archives, the host machine must be Pro // enabled and relevant Pro services must be enabled. The following commands From 94e7fe5a970795070512f6ee0e267e1b97e38e08 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 16:59:03 +0600 Subject: [PATCH 46/60] doc: add notes in README about the support of Pro --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/README.md b/README.md index 4e448125..5543a2b5 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,53 @@ provided packages and install only the desired slices into the *myrootfs* folder, according to the slice definitions available in the ["ubuntu-22.04" chisel-releases branch](). +## Chisel support for Pro archives + +Chisel can also fetch and install packages from Ubuntu Pro archives. For this, +the archive has to be defined with the `archives..pro` field in +chisel.yaml and its credentials have to be made available to Chisel. + + +```yaml +# chisel.yaml +format: v1 +archives: + : + pro: + ... +... +``` + +The archive URLs are hardcoded and varies based on the `pro` value. Chisel +currently supports the following Pro archives: + +| `pro` value | Archive URL | Related Ubuntu Pro service | +| - | - | - | +| fips | https://esm.ubuntu.com/fips/ubuntu | fips | +| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates | +| apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | +| infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | + +Since Pro archives require authentication, Chisel will support using credentials +from the `/etc/apt/auth.conf.d/` directory. This means chiselling from a Pro +archive is only possible if the host is also Pro (or equipped with the Pro +credentials). The default credentials directory - `/etc/apt/auth.conf.d/` - can +be configured by setting the environment variable `CHISEL_AUTH_DIR`. Chisel must +have read permission for the necessary credentials files. + +The format of these files is documented in the +[apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html) +man page. Below is a snippet of the `/etc/apt/auth.conf.d/90ubuntu-advantage` +file from a host with the `fips-updates` and `infra` archives enabled: + +``` +machine esm.ubuntu.com/infra/ubuntu/ login bearer password +machine esm.ubuntu.com/fips-updates/ubuntu/ login bearer password +``` + +_Note that the strings and are +just placeholders for real bearer tokens._ + ## Reference ### Chisel releases From 255417044b500179afa7e5a6f32a8ee073152762 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 17:23:50 +0600 Subject: [PATCH 47/60] test(archive): add a Pro archive test with bad creds --- internal/archive/archive_test.go | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index df51b3b5..adf999de 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -574,6 +574,7 @@ func (s *S) TestProArchives(c *C) { c.Skip("--real-pro-archive not provided") } s.runRealArchiveTests(c, proArchiveTests) + s.testProArchiveBadCreds(c) } func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { @@ -780,3 +781,35 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) } + +func (s *S) testProArchiveBadCreds(c *C) { + c.Logf("Cannot fetch Pro packages with bad credentials") + + credsDir := c.MkDir() + restore := fakeEnv("CHISEL_AUTH_DIR", credsDir) + defer restore() + + confFile := filepath.Join(credsDir, "credentials") + contents := "machine esm.ubuntu.com/fips/ubuntu/ login bearer password invalid" + err := os.WriteFile(confFile, []byte(contents), 0600) + c.Assert(err, IsNil) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: "fips", + PubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, + } + + // The archive can be "opened" without any credentials since the dists/ path + // containing InRelease files, does not require any credentials. + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) + + _, _, err = testArchive.Fetch("openssh-client") + c.Assert(err, ErrorMatches, `cannot find archive data`) +} From 3ff090da52f794f05e5490e1664550f19084604b Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 13:05:36 +0000 Subject: [PATCH 48/60] snap: add pro credentials access plug --- snap/snapcraft.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ee1c284c..6ec13a66 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -34,13 +34,15 @@ parts: plugin: go source: . build-snaps: - - go/1.21/stable + - go/1.23/stable build-environment: - CGO_ENABLED: 0 - GOFLAGS: -trimpath -ldflags=-w -ldflags=-s override-build: | go generate ./cmd craftctl default + stage: + - -bin/chrorder chisel-release-data: plugin: nil @@ -56,9 +58,17 @@ parts: craftctl set grade="$grade" after: [chisel] +plugs: + pro-credentials: + interface: system-files + read: + - /etc/apt/auth.conf.d + - /etc/apt/auth.conf.d/90ubuntu-advantage + apps: chisel: command: bin/chisel plugs: - network - home + - pro-credentials From 68206e5710cc9c76a0f71f24a9533dd30d08ca15 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 21:13:43 +0600 Subject: [PATCH 49/60] fix(archive): change error message for 401 response --- internal/archive/archive.go | 4 ++- internal/archive/archive_test.go | 60 +++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 1e3ab67a..18717e1d 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -394,7 +394,9 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea switch resp.StatusCode { case 200: // ok - case 401, 404: + case 401: + return nil, fmt.Errorf("cannot fetch from %s: authorization required", index.archive.baseURL) + case 404: return nil, fmt.Errorf("cannot find archive data") default: return nil, fmt.Errorf("error from archive: %v", resp.Status) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index adf999de..eaa1f215 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -404,14 +404,14 @@ func (s *httpSuite) TestProArchives(c *C) { for pro, info := range archive.ProArchiveInfo { s.base = info.BaseURL - s.prepareArchiveAdjustRelease("jammy", "22.04", "amd64", []string{"main", "universe"}, setLabel(info.Label)) + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) options := archive.Options{ Label: "ubuntu", - Version: "22.04", + Version: "20.04", Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, + Suites: []string{"focal"}, + Components: []string{"main"}, CacheDir: c.MkDir(), Pro: pro, PubKeys: []*packet.PublicKey{s.pubKey}, @@ -431,20 +431,56 @@ func (s *httpSuite) TestProArchives(c *C) { defer restoreDo() s.base = "http://archive.ubuntu.com/ubuntu/" - s.prepareArchive("jammy", "22.04", "amd64", []string{"main", "universe"}) + s.prepareArchive("focal", "20.04", "amd64", []string{"main"}) options := archive.Options{ Label: "ubuntu", - Version: "22.04", + Version: "20.04", Arch: "amd64", - Suites: []string{"jammy"}, - Components: []string{"main", "universe"}, + Suites: []string{"focal"}, + Components: []string{"main"}, CacheDir: c.MkDir(), PubKeys: []*packet.PublicKey{s.pubKey}, } _, err = archive.Open(&options) c.Assert(err, IsNil) + + // Test Pro archives with bad credentials. + do = func(req *http.Request) (*http.Response, error) { + _, ok := req.Header["Authorization"] + c.Assert(ok, Equals, true) + if strings.Contains(req.URL.String(), "/pool/") { + s.status = 401 + } else { + s.status = 200 + } + return s.Do(req) + } + restoreDo = archive.FakeDo(do) + defer restoreDo() + + for pro, info := range archive.ProArchiveInfo { + s.base = info.BaseURL + s.prepareArchiveAdjustRelease("focal", "20.04", "amd64", []string{"main"}, setLabel(info.Label)) + + options := archive.Options{ + Label: "ubuntu", + Version: "20.04", + Arch: "amd64", + Suites: []string{"focal"}, + Components: []string{"main"}, + CacheDir: c.MkDir(), + Pro: pro, + PubKeys: []*packet.PublicKey{s.pubKey}, + } + + testArchive, err := archive.Open(&options) + c.Assert(err, IsNil) + + _, _, err = testArchive.Fetch("mypkg1") + c.Assert(err, ErrorMatches, `cannot fetch from .*: authorization required`) + } } type verifyArchiveReleaseTest struct { @@ -569,12 +605,12 @@ func (s *S) TestRealArchive(c *C) { s.runRealArchiveTests(c, realArchiveTests) } -func (s *S) TestProArchives(c *C) { +func (s *S) TestRealProArchives(c *C) { if !*proArchiveFlag { c.Skip("--real-pro-archive not provided") } s.runRealArchiveTests(c, proArchiveTests) - s.testProArchiveBadCreds(c) + s.testRealProArchiveBadCreds(c) } func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { @@ -782,7 +818,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) { s.checkArchitecture(c, arch, filepath.Join(extractDir, "binary")) } -func (s *S) testProArchiveBadCreds(c *C) { +func (s *S) testRealProArchiveBadCreds(c *C) { c.Logf("Cannot fetch Pro packages with bad credentials") credsDir := c.MkDir() @@ -811,5 +847,5 @@ func (s *S) testProArchiveBadCreds(c *C) { c.Assert(err, IsNil) _, _, err = testArchive.Fetch("openssh-client") - c.Assert(err, ErrorMatches, `cannot find archive data`) + c.Assert(err, ErrorMatches, `cannot fetch from .*: authorization required`) } From 0ee1deb698998c2dc2835d7a8a707e21ef7aa62b Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 21:18:09 +0600 Subject: [PATCH 50/60] doc: update README text per suggestion --- README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5543a2b5..1efb007f 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,7 @@ archives: ... ``` -The archive URLs are hardcoded and varies based on the `pro` value. Chisel -currently supports the following Pro archives: +Chisel currently supports the following Pro archives: | `pro` value | Archive URL | Related Ubuntu Pro service | | - | - | - | @@ -94,12 +93,13 @@ currently supports the following Pro archives: | apps | https://esm.ubuntu.com/apps/ubuntu | esm-apps | | infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | -Since Pro archives require authentication, Chisel will support using credentials -from the `/etc/apt/auth.conf.d/` directory. This means chiselling from a Pro -archive is only possible if the host is also Pro (or equipped with the Pro -credentials). The default credentials directory - `/etc/apt/auth.conf.d/` - can -be configured by setting the environment variable `CHISEL_AUTH_DIR`. Chisel must -have read permission for the necessary credentials files. +Authentication to Pro archives requires that the host is Pro or it is equipped +with the Pro credentials. + +By default, Chisel will support using credentials from the +`/etc/apt/auth.conf.d/` directory, but this location can be configured using the +environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have read +permission for the necessary credentials files. The format of these files is documented in the [apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html) @@ -111,9 +111,6 @@ machine esm.ubuntu.com/infra/ubuntu/ login bearer password machine esm.ubuntu.com/fips-updates/ubuntu/ login bearer password ``` -_Note that the strings and are -just placeholders for real bearer tokens._ - ## Reference ### Chisel releases From 7ff02acae7104ce7459de7773a4abebd72d81c2c Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 21:40:52 +0600 Subject: [PATCH 51/60] test(archive): remove useless test --- internal/archive/archive_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index eaa1f215..bfba2d19 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -336,8 +336,6 @@ func (s *httpSuite) TestArchiveLabels(c *C) { label string err string }{{ - summary: "No labels", - }, { summary: "Ubuntu label", label: "Ubuntu", }, { From 47c92d9f4a5e59da7a244a0c15cf3292c9c26a16 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Wed, 16 Oct 2024 21:45:05 +0600 Subject: [PATCH 52/60] fix: use archive name in error messages instead of URL --- internal/archive/archive.go | 2 +- internal/archive/archive_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 18717e1d..57780657 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -395,7 +395,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea case 200: // ok case 401: - return nil, fmt.Errorf("cannot fetch from %s: authorization required", index.archive.baseURL) + return nil, fmt.Errorf("cannot fetch from %q: authorization required", index.label) case 404: return nil, fmt.Errorf("cannot find archive data") default: diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index bfba2d19..5b91e75c 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -477,7 +477,7 @@ func (s *httpSuite) TestProArchives(c *C) { c.Assert(err, IsNil) _, _, err = testArchive.Fetch("mypkg1") - c.Assert(err, ErrorMatches, `cannot fetch from .*: authorization required`) + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": authorization required`) } } @@ -845,5 +845,5 @@ func (s *S) testRealProArchiveBadCreds(c *C) { c.Assert(err, IsNil) _, _, err = testArchive.Fetch("openssh-client") - c.Assert(err, ErrorMatches, `cannot fetch from .*: authorization required`) + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": authorization required`) } From db539fd6866b11e238444144efd9ec3dbaa544db Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 15:51:45 +0600 Subject: [PATCH 53/60] fix(slicer): keep uniform error msg for no valid archives https://github.com/letFunny/chisel/pull/13#discussion_r1804302927 --- internal/slicer/slicer.go | 3 --- internal/slicer/slicer_test.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index bcad12e7..2991e253 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -86,9 +86,6 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - if len(options.Archives) == 0 { - return fmt.Errorf("no valid archives defined") - } pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d11ce3f8..4974d623 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1467,7 +1467,7 @@ var slicerTests = []slicerTest{{ contents: `, }, - error: `no valid archives defined`, + error: `cannot find package "test-package" in archive\(s\)`, }} var defaultChiselYaml = ` From 334d55a808a1cf05cc01a3f365a7387fcfc04da8 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 15:52:59 +0600 Subject: [PATCH 54/60] fix(archive): update 401 response error message --- internal/archive/archive.go | 2 +- internal/archive/archive_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 57780657..e2f4b4bb 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -395,7 +395,7 @@ func (index *ubuntuIndex) fetch(suffix, digest string, flags fetchFlags) (io.Rea case 200: // ok case 401: - return nil, fmt.Errorf("cannot fetch from %q: authorization required", index.label) + return nil, fmt.Errorf("cannot fetch from %q: unauthorized", index.label) case 404: return nil, fmt.Errorf("cannot find archive data") default: diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 5b91e75c..fa6dbb37 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -477,7 +477,7 @@ func (s *httpSuite) TestProArchives(c *C) { c.Assert(err, IsNil) _, _, err = testArchive.Fetch("mypkg1") - c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": authorization required`) + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) } } @@ -845,5 +845,5 @@ func (s *S) testRealProArchiveBadCreds(c *C) { c.Assert(err, IsNil) _, _, err = testArchive.Fetch("openssh-client") - c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": authorization required`) + c.Assert(err, ErrorMatches, `cannot fetch from "ubuntu": unauthorized`) } From 2608ee3ab302dc5dde18995c7f437e2bbc44ed21 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 16:03:45 +0600 Subject: [PATCH 55/60] fix(archive): validate pro value when opening archive --- internal/archive/archive.go | 7 ++++++- internal/archive/archive_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index e2f4b4bb..8deb9d71 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -173,7 +173,7 @@ var proArchiveInfo = map[string]struct { } // archiveURL returns the archive base URL depending on the "pro" value and -// selected architecture "arch". The "pro" value must be pre-validated. +// selected architecture "arch". func archiveURL(pro, arch string) string { if pro != "" { return proArchiveInfo[pro].BaseURL @@ -194,6 +194,11 @@ func openUbuntu(options *Options) (Archive, error) { if len(options.Version) == 0 { return nil, fmt.Errorf("archive options missing version") } + if options.Pro != "" { + if _, ok := proArchiveInfo[options.Pro]; !ok { + return nil, fmt.Errorf("invalid pro value: %q", options.Pro) + } + } baseURL := archiveURL(options.Pro, options.Arch) var creds *credentials diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index fa6dbb37..ea05a5f2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -181,6 +181,16 @@ var optionErrorTests = []optionErrorTest{{ Components: []string{"main", "other"}, }, error: `invalid package architecture: foo`, +}, { + options: archive.Options{ + Label: "ubuntu", + Version: "22.04", + Arch: "amd64", + Suites: []string{"jammy"}, + Components: []string{"main", "other"}, + Pro: "invalid", + }, + error: `invalid pro value: "invalid"`, }} func (s *httpSuite) TestOptionErrors(c *C) { From 15d0805070b6d1a7c56b3933bf30b719753b2c39 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 17:14:02 +0600 Subject: [PATCH 56/60] fix(archive): tweak the invalid pro value error msg --- internal/archive/archive.go | 2 +- internal/archive/archive_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 8deb9d71..87a8abdd 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -196,7 +196,7 @@ func openUbuntu(options *Options) (Archive, error) { } if options.Pro != "" { if _, ok := proArchiveInfo[options.Pro]; !ok { - return nil, fmt.Errorf("invalid pro value: %q", options.Pro) + return nil, fmt.Errorf("archive options has invalid pro value: %q", options.Pro) } } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index ea05a5f2..482f07f6 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -190,7 +190,7 @@ var optionErrorTests = []optionErrorTest{{ Components: []string{"main", "other"}, Pro: "invalid", }, - error: `invalid pro value: "invalid"`, + error: `archive options has invalid pro value: "invalid"`, }} func (s *httpSuite) TestOptionErrors(c *C) { From 39fd79ddce0525de899538b3026f322546ddfa48 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 17:17:25 +0600 Subject: [PATCH 57/60] Revert "fix(archive): tweak the invalid pro value error msg" This reverts commit 15d0805070b6d1a7c56b3933bf30b719753b2c39. --- internal/archive/archive.go | 2 +- internal/archive/archive_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 87a8abdd..8deb9d71 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -196,7 +196,7 @@ func openUbuntu(options *Options) (Archive, error) { } if options.Pro != "" { if _, ok := proArchiveInfo[options.Pro]; !ok { - return nil, fmt.Errorf("archive options has invalid pro value: %q", options.Pro) + return nil, fmt.Errorf("invalid pro value: %q", options.Pro) } } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 482f07f6..ea05a5f2 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -190,7 +190,7 @@ var optionErrorTests = []optionErrorTest{{ Components: []string{"main", "other"}, Pro: "invalid", }, - error: `archive options has invalid pro value: "invalid"`, + error: `invalid pro value: "invalid"`, }} func (s *httpSuite) TestOptionErrors(c *C) { From 6e14e4e82a1a2fa8f7164d16d7f4b74617cc8b8a Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 19:48:04 +0600 Subject: [PATCH 58/60] snap: keep using go 1.21 to build chisel --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 6ec13a66..958a5970 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -34,7 +34,7 @@ parts: plugin: go source: . build-snaps: - - go/1.23/stable + - go/1.21/stable build-environment: - CGO_ENABLED: 0 - GOFLAGS: -trimpath -ldflags=-w -ldflags=-s From 2471a6720164273feb023116b8740240e4fb1479 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Thu, 17 Oct 2024 22:25:51 +0600 Subject: [PATCH 59/60] fix: address Alberto's comments https://github.com/letFunny/chisel/pull/13#pullrequestreview-2375148055 --- .github/workflows/tests.yaml | 1 - README.md | 12 +++++------- internal/archive/archive.go | 4 ++-- internal/archive/archive_test.go | 26 +++++++++++++------------- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4cf514ea..7510cfc8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -75,7 +75,6 @@ jobs: # other services, add the credentials back. sudo pro enable fips --assume-yes sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds - # This will disable the fips service. sudo pro enable fips-updates esm-apps esm-infra --assume-yes # Add the fips credentials back. diff --git a/README.md b/README.md index 1efb007f..63243312 100644 --- a/README.md +++ b/README.md @@ -94,14 +94,12 @@ Chisel currently supports the following Pro archives: | infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | Authentication to Pro archives requires that the host is Pro or it is equipped -with the Pro credentials. +with the Pro credentials. By default, Chisel will support using credentials +from the `/etc/apt/auth.conf.d/` directory, but this location can be configured +using the environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have +read permission for the necessary credentials files. -By default, Chisel will support using credentials from the -`/etc/apt/auth.conf.d/` directory, but this location can be configured using the -environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have read -permission for the necessary credentials files. - -The format of these files is documented in the +The format of the files is documented in the [apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html) man page. Below is a snippet of the `/etc/apt/auth.conf.d/90ubuntu-advantage` file from a host with the `fips-updates` and `infra` archives enabled: diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 8deb9d71..9b81bda6 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -239,8 +239,8 @@ func openUbuntu(options *Options) (Archive, error) { } release = index.release if !index.supportsArch(options.Arch) { - // Release does not support the specified architecture. Do - // not add indexes of this release. + // Release does not support the specified architecture, do + // not add any of its indexes. logf("Warning: ignoring %s %s %s suite (unsupported arch %s)...", index.proSuffixedLabel(), index.version, index.suite, options.Arch) break diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index ea05a5f2..cf586bab 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -627,10 +627,10 @@ func (s *S) runRealArchiveTests(c *C, tests []realArchiveTest) { allArch = append(allArch, arch) } for _, test := range tests { - if len(test.architectures) == 0 { - test.architectures = allArch + if len(test.archs) == 0 { + test.archs = allArch } - for _, arch := range test.architectures { + for _, arch := range test.archs { s.testOpenArchiveArch(c, test, arch) } } @@ -643,7 +643,7 @@ type realArchiveTest struct { components []string pro string archivePubKeys []*packet.PublicKey - architectures []string + archs []string pkg string path string } @@ -681,7 +681,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "fips", archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "openssh-client", path: "/usr/bin/ssh", }, { @@ -691,7 +691,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "fips-updates", archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "openssh-client", path: "/usr/bin/ssh", }, { @@ -701,7 +701,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "apps", archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", }, { @@ -711,7 +711,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "infra", archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", }, { @@ -721,7 +721,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "fips-updates", archivePubKeys: []*packet.PublicKey{keyUbuntuFIPSv1.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "openssh-client", path: "/usr/bin/ssh", }, { @@ -731,7 +731,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "apps", archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", }, { @@ -741,7 +741,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "infra", archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", }, { @@ -751,7 +751,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "apps", archivePubKeys: []*packet.PublicKey{keyUbuntuApps.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", }, { @@ -761,7 +761,7 @@ var proArchiveTests = []realArchiveTest{{ components: []string{"main"}, pro: "infra", archivePubKeys: []*packet.PublicKey{keyUbuntuESMv2.PubKey}, - architectures: []string{"amd64"}, + archs: []string{"amd64"}, pkg: "hello", path: "/usr/bin/hello", }} From 2188322a5ffd79e4f1f60fc5581091fa89c37c82 Mon Sep 17 00:00:00 2001 From: Rafid Bin Mostofa Date: Fri, 18 Oct 2024 14:28:59 +0600 Subject: [PATCH 60/60] docs: remove double space --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 63243312..cf5b5f8b 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Chisel currently supports the following Pro archives: | infra | https://esm.ubuntu.com/infra/ubuntu | esm-infra | Authentication to Pro archives requires that the host is Pro or it is equipped -with the Pro credentials. By default, Chisel will support using credentials +with the Pro credentials. By default, Chisel will support using credentials from the `/etc/apt/auth.conf.d/` directory, but this location can be configured using the environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have read permission for the necessary credentials files.