CI test: Pro archives support

.github/workflows/spread.yml

@@ -27,6 +27,8 @@ jobs:
           go-version: '>=1.17.0'
 
       - name: Build and run spread
+        env:
+          PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
         run: |
           (cd _spread/cmd/spread && go build)
           _spread/cmd/spread/spread -v focal jammy mantic noble

.github/workflows/tests.yaml

@@ -43,3 +43,46 @@ jobs:
         with:
           name: chisel-test-coverage.html
           path: ./*.html
+
+  real-archive-tests:
+    # Do not change to newer releases as "fips" may not be available there.
+    runs-on: ubuntu-20.04
+    name: Real Archive Tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run real archive tests
+        env:
+          PRO_TOKEN: ${{ secrets.PRO_TOKEN }}
+        run: |
+          set -ex
+
+          detach() {
+            sudo pro detach --assume-yes || true
+            sudo rm -f /etc/apt/auth.conf.d/90ubuntu-advantage
+          }
+          trap detach EXIT
+
+          # Attach pro token and enable services
+          sudo pro attach ${PRO_TOKEN} --no-auto-enable
+
+          # Cannot enable fips and fips-updates at the same time.
+          # Hack: enable fips, copy the credentials and then after enabling
+          # other services, add the credentials back.
+          sudo pro enable fips --assume-yes
+          sudo cp /etc/apt/auth.conf.d/90ubuntu-advantage /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds
+          # This will disable the fips service.
+          sudo pro enable fips-updates esm-apps esm-infra --assume-yes
+          # Add the fips credentials back.
+          sudo sh -c 'cat /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds >> /etc/apt/auth.conf.d/90ubuntu-advantage'
+          sudo rm /etc/apt/auth.conf.d/90ubuntu-advantage.fips-creds
+
+          # Make apt credentials accessible to USER.
+          sudo setfacl -m u:$USER:r /etc/apt/auth.conf.d/90ubuntu-advantage
+
+          # Run tests on Pro and non-Pro real archives.
+          go test ./internal/archive/ -v --real-archive --real-pro-archive

README.md

@@ -67,6 +67,48 @@ provided packages and install only the desired slices into the *myrootfs*
 folder, according to the slice definitions available in the
 ["ubuntu-22.04" chisel-releases branch](<https://github.com/canonical/chisel-releases/tree/ubuntu-22.04>).
 
+## Chisel support for Pro archives
+
+Chisel can also fetch and install packages from Ubuntu Pro archives. For this,
+the archive has to be defined with the `archives.<archive>.pro` field in
+chisel.yaml and its credentials have to be made available to Chisel.
+
+
+```yaml
+# chisel.yaml
+format: v1
+archives:
+  <archive-name>:
+    pro: <value>
+    ...
+...
+```
+
+Chisel currently supports the following Pro archives:
+
+| `pro` value | Archive URL | Related Ubuntu Pro service |
+| - | - | - |
+| fips         | https://esm.ubuntu.com/fips/ubuntu         | fips         |
+| fips-updates | https://esm.ubuntu.com/fips-updates/ubuntu | fips-updates |
+| apps         | https://esm.ubuntu.com/apps/ubuntu         | esm-apps     |
+| infra        | https://esm.ubuntu.com/infra/ubuntu        | esm-infra    |
+
+Authentication to Pro archives requires that the host is Pro or it is equipped
+with the Pro credentials. By default, Chisel will support using credentials
+from the `/etc/apt/auth.conf.d/` directory, but this location can be configured
+using the environment variable `CHISEL_AUTH_DIR`. Note that Chisel must have
+read permission for the necessary credentials files.
+
+The format of the files is documented in the
+[apt_auth.conf(5)](https://manpages.debian.org/testing/apt/apt_auth.conf.5.en.html)
+man page. Below is a snippet of the `/etc/apt/auth.conf.d/90ubuntu-advantage`
+file from a host with the `fips-updates` and `infra` archives enabled:
+
+```
+machine esm.ubuntu.com/infra/ubuntu/ login bearer password <infra-token>
+machine esm.ubuntu.com/fips-updates/ubuntu/ login bearer password <fips-updates-token>
+```
+
 ## Reference
 
 ### Chisel releases

cmd/chisel/cmd_cut.go

@@ -70,10 +70,15 @@ func (cmd *cmdCut) Execute(args []string) error {
 			Arch:       cmd.Arch,
 			Suites:     archiveInfo.Suites,
 			Components: archiveInfo.Components,
+			Pro:        archiveInfo.Pro,
 			CacheDir:   cache.DefaultDir("chisel"),
 			PubKeys:    archiveInfo.PubKeys,
 		})
 		if err != nil {
+			if err == archive.ErrCredentialsNotFound {
+				logf("Ignoring archive %q (credentials not found)...", archiveName)
+				continue
+			}
 			return err
 		}
 		archives[archiveName] = openArchive

cmd/chisel/cmd_find_test.go

@@ -33,8 +33,6 @@ func makeSamplePackage(pkg string, slices []string) *setup.Package {
 }
 
 var sampleRelease = &setup.Release{
-	DefaultArchive: "ubuntu",
-
 	Archives: map[string]*setup.Archive{
 		"ubuntu": {
 			Name:       "ubuntu",

cmd/chisel/cmd_info_test.go

@@ -25,7 +25,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -37,7 +36,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg2"},
 	stdout: `
 		package: mypkg2
-		archive: ubuntu
 		slices:
 			myslice:
 				contents:
@@ -49,7 +47,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice2", "mypkg1_myslice1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -65,7 +62,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1", "mypkg2", "mypkg1_myslice2"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -76,7 +72,6 @@ var infoTests = []infoTest{{
 					- mypkg2_myslice
 		---
 		package: mypkg2
-		archive: ubuntu
 		slices:
 			myslice:
 				contents:
@@ -88,7 +83,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1", "mypkg1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -104,7 +98,6 @@ var infoTests = []infoTest{{
 	query:   []string{"mypkg1_myslice1", "mypkg1_myslice1", "mypkg1_myslice1"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:
@@ -121,7 +114,6 @@ var infoTests = []infoTest{{
 	query:   []string{"foo", "mypkg1_myslice1", "bar_foo"},
 	stdout: `
 		package: mypkg1
-		archive: ubuntu
 		slices:
 			myslice1:
 				contents:

cmd/chisel/main.go

@@ -327,6 +327,7 @@ func run() error {
 	deb.SetLogger(log.Default())
 	setup.SetLogger(log.Default())
 	slicer.SetLogger(log.Default())
+	SetLogger(log.Default())
 
 	parser := Parser()
 	xtra, err := parser.Parse()