update use of auth.json credentials for import, add test of existing image import behavior

Makefile

@@ -47,7 +47,7 @@ BATS = $(TOOLS_D)/bin/bats
 BATS_VERSION := v1.10.0
 # OCI registry
 ZOT := $(TOOLS_D)/bin/zot
-ZOT_VERSION := v2.1.0
+ZOT_VERSION := v2.1.8
 UMOCI := $(TOOLS_D)/bin/umoci
 UMOCI_VERSION := main
 
@@ -124,7 +124,7 @@ go-test:
 	go tool cover -html coverage.txt  -o $(HACK_D)/coverage.html
 
 .PHONY: download-tools
-download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI)
+download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) $(SKOPEO)
 
 $(GOLANGCI_LINT):
 	@mkdir -p $(dir $@)

doc/stacker_yaml.md

@@ -40,10 +40,12 @@ Some directives are irrelevant depending on the type. Supported types are:
 
 - `url` is required
 
-- `insecure` is optional. 
+- `insecure` is optional. If unspecified, the default is is `false`.
 
 When `insecure` is specified, stacker attempts to connect via http instead of
-https to the Docker Hub.
+https to the image registry in the `url`.
+
+When `insecure` is false, Stacker can authenticate to the image registry using HTTP Basic Authentication. See the section "Credential Handling" for further details.
 
 #### `type: tar`
 
@@ -86,21 +88,8 @@ will NOT update this file unless the cache is cleared, to avoid excess network
 usage. That means that updates after the first time stacker downloads the file
 will not be reflected. To force re-downloading, use `stacker build --no-cache`.
 
-Stacker supports Basic Authentication for imports from an HTTP server. It will
-attempt to find credentials matching the hostname of the URL in the `auth.json`
-file documented at
-[containers-auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md)
-
-So for example, this `auth.json` file will allow authentication to example.com
-with the username and password of `aw:yeah` (encoded as base64).
+Stacker supports Basic Authentication for imports from an HTTPS server. See the section "Credential handling" for details.
 
-```json
-{
-  "auths": {
-    "example.com": "YXc6eWVhaA=="
-  }
-}
-```
 
     stacker://$name/path/to/file
 
@@ -300,7 +289,7 @@ defaults to the host operating system if not specified.
 built for, for example, `amd64`, `arm64`, etc. It is an optional field and it
 defaults to the host machine architecture if not specified.
 
-### Substitution Syntax
+## Substitution Syntax
 
 Before the yaml is parsed, stacker performs substitution on placeholders in the
 file of the format `${{VAR}}` or `${{VAR:default}}`. See [Substitution
@@ -355,3 +344,35 @@ available for reference:
       my-build:
         run: echo "Your layer is ${STACKER_LAYER_NAME}"
       ```
+
+## Credential Handling
+
+Stacker uses OCI standard credential storage for authenticating to both container registries (for `from` images) and https servers (for `import`s).
+
+Stacker will look for credentials in the `auth.json` file documented at
+[containers-auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md).
+The longest matching substring of the import URL is used.
+
+So for example, this `auth.json` file will allow authentication to example.com
+with the username and password of `aw:yeah` (encoded as base64). For a URL
+`https://example.com:8080/reg1/p1/path/to/file.txt` or any other file under
+`reg1/p1` on that host, this file specifies the alternate creds `arr:narr`:
+
+```json
+{
+  "auths": {
+    "example.com": "YXc6eWVhaA==",
+    "example.com:8080/reg1/p1": "YXJyOm5hcnI="
+  }
+}
+```
+
+NOTE: due to the way the subpaths are broken up to search for the longest
+matching subpath, the key in `auth.json` must not have a trailing slash.
+`reg1/p1/` will not match any file's path.`
+
+### Generating auth.json
+
+`auth.json` is simple enough to generate by hand, but it can also be created and
+updated by running `skopeo login $registry_url` for OCI compatible container
+image registries.

pkg/stacker/network.go

@@ -1,6 +1,7 @@
 package stacker
 
 import (
+	"fmt"
 	"io"
 	"io/fs"
 	"net/http"
@@ -85,12 +86,13 @@ func Download(cacheDir string, remoteUrl string, progress bool, expectedHash, re
 	if err != nil {
 		return "", err
 	}
-
-	creds, err := config.GetCredentials(&types.SystemContext{}, u.Hostname())
+	key := fmt.Sprintf("%s%s", u.Host, u.Path)
+	log.Infof("searching creds for key %q", key)
+	creds, err := config.GetCredentials(&types.SystemContext{}, key)
 	if err != nil {
 		log.Infof("credentials not found for host %s - reason:%s continuing without creds", u.Host, err)
 	}
-
+	log.Infof("found creds for key %q: %+v", key, creds)
 	request.SetBasicAuth(creds.Username, creds.Password)
 
 	client := &http.Client{}

test/basic-auth.bats

@@ -0,0 +1,36 @@
+load helpers
+
+function setup() {
+    stacker_setup
+    zot_setup_auth
+    cat > stacker.yaml <<EOF
+busybox:
+    from:
+        type: docker
+        url: docker://${ZOT_HOST}:${ZOT_PORT}/busybox:latest
+EOF
+
+}
+
+function teardown() {
+    zot_teardown
+    cleanup
+}
+
+@test "from: authenticated zot works" {
+    require_privilege priv
+
+    export XDG_RUNTIME_DIR=$TEST_TMPDIR
+    mkdir -p $TEST_TMPDIR/containers/
+    cat > $TEST_TMPDIR/containers/auth.json <<EOF
+{
+     "auths": {
+          "${ZOT_HOST}:${ZOT_PORT}": {"auth": "aWFtOmNhcmVmdWw="}
+     }
+}
+EOF
+    export SSL_CERT_FILE=$BATS_SUITE_TMPDIR/ca.crt
+    skopeo copy --dest-creds "iam:careful" oci:${BUSYBOX_OCI} docker://${ZOT_HOST}:${ZOT_PORT}/busybox:latest
+
+    stacker --debug build
+}

test/bom.bats

@@ -155,7 +155,6 @@ EOF
     # sbom for this image
     [ -f .stacker/artifacts/second/second.json ]
     if [ -n "${ZOT_HOST}${ZOT_PORT}" ]; then
-      zot_setup
       stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest --substitute CENTOS_OCI=${CENTOS_OCI}
       refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/first:latest --format "{{json .}}" | jq '.referrer | length')
       [ $refs -eq 2 ]
@@ -165,7 +164,6 @@ EOF
       [ $refs -eq 2 ]
       refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/second:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID')
       [ $refs == \"SPDXRef-DOCUMENT\" ]
-      zot_teardown
     fi
     stacker clean
 }
@@ -205,7 +203,6 @@ EOF
     # sbom for this image
     [ -f .stacker/artifacts/bom-alpine/bom-alpine.json ]
     if [ -n "${ZOT_HOST}${ZOT_PORT}" ]; then
-      zot_setup
       stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest --substitute ALPINE_OCI=${ALPINE_OCI}
       refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/bom-alpine:latest --format "{{json .}}" | jq '.referrer | length')
       [ $refs -eq 2 ]
@@ -216,7 +213,7 @@ EOF
 }
 
 @test "pull boms if published" {
-  #skip_slow_test
+
   cat > stacker.yaml <<EOF
 parent:
     from:
@@ -261,13 +258,15 @@ EOF
     if [ -n "${ZOT_HOST}${ZOT_PORT}" ]; then
       stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest
       refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/parent:latest --format "{{json .}}" | jq '.referrer | length')
-      [ $refs -eq 2 ]
+      [[ $refs -eq 2 ]]
       refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/parent:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID')
-      [ $refs == \"SPDXRef-DOCUMENT\" ]
+      [[ $refs == \"SPDXRef-DOCUMENT\" ]]
+
     fi
 
     if [ -z "${ZOT_HOST}${ZOT_PORT}" ]; then
-      skip "second half of test requires running zot"
+      echo "# skipping second half of test $BATS_SUITE_TEST_NUMBER because it requires running zot" >&3
+      return 0
     fi
 
   cat > stacker.yaml <<EOF
@@ -300,13 +299,14 @@ EOF
     [ -f .stacker/artifacts/child/inventory.json ]
     # sbom for this image
     [ -f .stacker/artifacts/child/child.json ]
-    if [ -n "${ZOT_HOST}${ZOT_PORT}" ]; then
-      stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest
-      refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/child:latest --format "{{json .}}" | jq '.referrer | length')
-      [ $refs -eq 2 ]
-      refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/child:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID')
-      [ $refs == \"SPDXRef-DOCUMENT\" ]
-    fi
+
+    # zot is running already, we exited early if it wasn't.
+    stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest
+    refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/child:latest --format "{{json .}}" | jq '.referrer | length')
+    [ $refs -eq 2 ]
+    refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/child:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID')
+    [ $refs == \"SPDXRef-DOCUMENT\" ]
+
     stacker clean
 }
 
@@ -455,13 +455,13 @@ EOF
     # sbom for this image
     [ -f .stacker/artifacts/second/second.json ]
     if [ -n "${ZOT_HOST}${ZOT_PORT}" ]; then
-      zot_setup
+
       stacker publish --skip-tls --url docker://${ZOT_HOST}:${ZOT_PORT} --tag latest --substitute CENTOS_OCI=${CENTOS_OCI}
       refs=$(regctl artifact tree ${ZOT_HOST}:${ZOT_PORT}/second:latest --format "{{json .}}" | jq '.referrer | length')
       [ $refs -eq 2 ]
       refs=$(regctl artifact get --subject ${ZOT_HOST}:${ZOT_PORT}/second:latest --filter-artifact-type "application/spdx+json" | jq '.SPDXID')
       [ $refs == \"SPDXRef-DOCUMENT\" ]
-      zot_teardown
+
     fi
     stacker clean
 }

test/helpers.bash

@@ -73,7 +73,7 @@ export ALPINE_OCI="$ROOT_DIR/test/alpine:edge"
 export BUSYBOX_OCI="$ROOT_DIR/test/busybox:latest"
 export CENTOS_OCI="$ROOT_DIR/test/centos:latest"
 export UBUNTU_OCI="$ROOT_DIR/test/ubuntu:latest"
-export PATH="$PATH:$ROOT_DIR/hack/tools/bin"
+export PATH="$ROOT_DIR/hack/tools/bin:$PATH"
 
 function sha() {
     echo $(sha256sum $1 | cut -f1 -d" ")
@@ -177,8 +177,7 @@ function cmp_files() {
     return 0
 }
 
-function zot_setup {
-  echo "# starting zot" >&3
+function write_plain_zot_config {
   cat > $TEST_TMPDIR/zot-config.json << EOF
 {
   "distSpecVersion": "1.1.0-dev",
@@ -192,23 +191,97 @@ function zot_setup {
     "port": "$ZOT_PORT"
   },
   "log": {
-    "level": "error"
+    "level": "debug",
+    "output": "$TEST_TMPDIR/zot.log"
   }
 }
 EOF
-	# start as a background task
+
+}
+
+function write_auth_zot_config {
+
+  htpasswd -Bbn iam careful >> $TEST_TMPDIR/htpasswd
+
+  cat > $TEST_TMPDIR/zot-config.json << EOF
+{
+  "distSpecVersion": "1.1.0-dev",
+  "storage": {
+    "rootDirectory": "$TEST_TMPDIR/zot",
+    "gc": true,
+    "dedupe": true
+  },
+  "http": {
+    "tls": {
+      "cert": "$BATS_SUITE_TMPDIR/server.cert",
+      "key": "$BATS_SUITE_TMPDIR/server.key"
+    },
+    "address": "$ZOT_HOST",
+    "port": "$ZOT_PORT",
+    "auth": {
+      "htpasswd": {
+        "path": "$TEST_TMPDIR/htpasswd"
+      }
+    },
+    "accessControl": {
+      "repositories": {
+        "**": {
+          "policies": [{
+              "users": [ "iam" ],
+              "actions": [ "read", "create", "update" ]
+          }]
+        }
+      }
+    }
+  },
+  "log": {
+    "level": "debug",
+    "output": "$TEST_TMPDIR/zot.log",
+    "audit": "$TEST_TMPDIR/zot-audit.log"
+  }
+}
+EOF
+
+}
+
+function zot_setup {
+    write_plain_zot_config
+    start_zot
+}
+
+function zot_setup_auth {
+    write_auth_zot_config
+    start_zot USE_TLS
+}
+
+function start_zot {
+  ZOT_USE_TLS=$1
+  echo "# starting zot at $ZOT_HOST:$ZOT_PORT" >&3
+  # start as a background task
+  zot verify $TEST_TMPDIR/zot-config.json
   zot serve $TEST_TMPDIR/zot-config.json &
   pid=$!
+
+  echo "zot is running at pid $pid"
+  cat $TEST_TMPDIR/zot.log
   # wait until service is up
   count=5
   up=0
+
   while [[ $count -gt 0 ]]; do
     if [ ! -d /proc/$pid ]; then
       echo "zot failed to start or died"
       exit 1
     fi
     up=1
-    curl -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0
+    if [[ -n $ZOT_USE_TLS ]]; then
+        echo "testing zot at https://$ZOT_HOST:$ZOT_PORT"
+        curl -v --cacert $BATS_SUITE_TMPDIR/ca.crt  -u "iam:careful" -f https://$ZOT_HOST:$ZOT_PORT/v2/ || up=0
+    else
+        echo "testing zot at http://$ZOT_HOST:$ZOT_PORT"
+        curl -v -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0
+    fi
+
     if [ $up -eq 1 ]; then break; fi
     sleep 1
     count=$((count - 1))
@@ -217,8 +290,15 @@ EOF
     echo "Timed out waiting for zot"
     exit 1
   fi
+
+  echo "# zot is up" >&3
   # setup a OCI client
-  regctl registry set --tls=disabled $ZOT_HOST:$ZOT_PORT
+  if [[ -n $ZOT_USE_TLS ]]; then
+      regctl registry set $ZOT_HOST:$ZOT_PORT
+  else
+      regctl registry set --tls=disabled $ZOT_HOST:$ZOT_PORT
+  fi
+
 }
 
 function zot_teardown {