From c85495301b2a9db3fca070e478129ec47fd3bbde Mon Sep 17 00:00:00 2001 From: Michael McCracken Date: Wed, 17 Sep 2025 16:36:54 -0700 Subject: [PATCH 1/4] ci: ensure we have new skopeo ensure we get the new skopeo into hack/tools/bin and then put that at the front of PATH Signed-off-by: Michael McCracken --- Makefile | 2 +- test/helpers.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4a23a502c..289cd6b25 100644 --- a/Makefile +++ b/Makefile @@ -124,7 +124,7 @@ go-test: go tool cover -html coverage.txt -o $(HACK_D)/coverage.html .PHONY: download-tools -download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) +download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) $(SKOPEO) $(GOLANGCI_LINT): @mkdir -p $(dir $@) diff --git a/test/helpers.bash b/test/helpers.bash index ac82cae43..60ac0826b 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -73,7 +73,7 @@ export ALPINE_OCI="$ROOT_DIR/test/alpine:edge" export BUSYBOX_OCI="$ROOT_DIR/test/busybox:latest" export CENTOS_OCI="$ROOT_DIR/test/centos:latest" export UBUNTU_OCI="$ROOT_DIR/test/ubuntu:latest" -export PATH="$PATH:$ROOT_DIR/hack/tools/bin" +export PATH="$ROOT_DIR/hack/tools/bin:$PATH" function sha() { echo $(sha256sum $1 | cut -f1 -d" ") From 2008757e32bbbe3d243a4d344c781c774f495761 Mon Sep 17 00:00:00 2001 From: Michael McCracken Date: Thu, 11 Sep 2025 17:39:02 -0700 Subject: [PATCH 2/4] test: auth.json creds works for image import This is a test to check existing behavior. Internally, without stacker needing to pass creds in the opts, containers/image uses GetCredentials() to look in auth.json for creds for calls to copy.Image(). this adds a test to cover this case using a zot configured to require auth. also generates certs for the zot in the test. some other cleanup in tests Signed-off-by: Michael McCracken --- Makefile | 2 +- test/basic-auth.bats | 36 +++++++++++++++++ test/bom.bats | 32 +++++++-------- test/helpers.bash | 92 ++++++++++++++++++++++++++++++++++++++++--- test/main.py | 2 +- test/setup_suite.bash | 52 ++++++++++++++++++++++++ 6 files changed, 192 insertions(+), 24 deletions(-) create mode 100644 test/basic-auth.bats diff --git a/Makefile b/Makefile index 289cd6b25..c409c3179 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ BATS = $(TOOLS_D)/bin/bats BATS_VERSION := v1.10.0 # OCI registry ZOT := $(TOOLS_D)/bin/zot -ZOT_VERSION := v2.1.0 +ZOT_VERSION := v2.1.8 UMOCI := $(TOOLS_D)/bin/umoci UMOCI_VERSION := main diff --git a/test/basic-auth.bats b/test/basic-auth.bats new file mode 100644 index 000000000..d31d97e6c --- /dev/null +++ b/test/basic-auth.bats @@ -0,0 +1,36 @@ +load helpers + +function setup() { + stacker_setup + zot_setup_auth + cat > stacker.yaml < $TEST_TMPDIR/containers/auth.json < stacker.yaml <&3 + return 0 fi cat > stacker.yaml <&3 +function write_plain_zot_config { cat > $TEST_TMPDIR/zot-config.json << EOF { "distSpecVersion": "1.1.0-dev", @@ -192,23 +191,97 @@ function zot_setup { "port": "$ZOT_PORT" }, "log": { - "level": "error" + "level": "debug", + "output": "$TEST_TMPDIR/zot.log" } } EOF - # start as a background task + +} + +function write_auth_zot_config { + + htpasswd -Bbn iam careful >> $TEST_TMPDIR/htpasswd + + cat > $TEST_TMPDIR/zot-config.json << EOF +{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "$TEST_TMPDIR/zot", + "gc": true, + "dedupe": true + }, + "http": { + "tls": { + "cert": "$BATS_SUITE_TMPDIR/server.cert", + "key": "$BATS_SUITE_TMPDIR/server.key" + }, + "address": "$ZOT_HOST", + "port": "$ZOT_PORT", + "auth": { + "htpasswd": { + "path": "$TEST_TMPDIR/htpasswd" + } + }, + "accessControl": { + "repositories": { + "**": { + "policies": [{ + "users": [ "iam" ], + "actions": [ "read", "create", "update" ] + }] + } + } + } + }, + "log": { + "level": "debug", + "output": "$TEST_TMPDIR/zot.log", + "audit": "$TEST_TMPDIR/zot-audit.log" + } +} +EOF + +} + +function zot_setup { + write_plain_zot_config + start_zot +} + +function zot_setup_auth { + write_auth_zot_config + start_zot USE_TLS +} + +function start_zot { + ZOT_USE_TLS=$1 + echo "# starting zot at $ZOT_HOST:$ZOT_PORT" >&3 + # start as a background task + zot verify $TEST_TMPDIR/zot-config.json zot serve $TEST_TMPDIR/zot-config.json & pid=$! + + echo "zot is running at pid $pid" + cat $TEST_TMPDIR/zot.log # wait until service is up count=5 up=0 + while [[ $count -gt 0 ]]; do if [ ! -d /proc/$pid ]; then echo "zot failed to start or died" exit 1 fi up=1 - curl -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 + if [[ -n $ZOT_USE_TLS ]]; then + echo "testing zot at https://$ZOT_HOST:$ZOT_PORT" + curl -v --cacert $BATS_SUITE_TMPDIR/ca.crt -u "iam:careful" -f https://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 + else + echo "testing zot at http://$ZOT_HOST:$ZOT_PORT" + curl -v -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 + fi + if [ $up -eq 1 ]; then break; fi sleep 1 count=$((count - 1)) @@ -217,8 +290,15 @@ EOF echo "Timed out waiting for zot" exit 1 fi + + echo "# zot is up" >&3 # setup a OCI client - regctl registry set --tls=disabled $ZOT_HOST:$ZOT_PORT + if [[ -n $ZOT_USE_TLS ]]; then + regctl registry set $ZOT_HOST:$ZOT_PORT + else + regctl registry set --tls=disabled $ZOT_HOST:$ZOT_PORT + fi + } function zot_teardown { diff --git a/test/main.py b/test/main.py index 0c269d7b5..52c4a7d8f 100755 --- a/test/main.py +++ b/test/main.py @@ -22,7 +22,7 @@ priv_to_test = [options.privilege_level] for priv in priv_to_test: - cmd = ["bats", "--jobs", str(options.jobs), "--tap", "--timing"] + cmd = ["bats", "--setup-suite-file", "./test/setup_suite.bash", "--jobs", str(options.jobs), "--tap", "--timing"] cmd.extend(options.tests) env = os.environ.copy() diff --git a/test/setup_suite.bash b/test/setup_suite.bash index df4899431..cb02fd3e4 100644 --- a/test/setup_suite.bash +++ b/test/setup_suite.bash @@ -1,6 +1,58 @@ #!/bin/bash +function write_certs { + pushd $BATS_SUITE_TMPDIR + + openssl req \ + -newkey rsa:2048 \ + -nodes \ + -days 3650 \ + -x509 \ + -keyout ca.key \ + -out ca.crt \ + -subj "/CN=*" + + openssl req \ + -newkey rsa:2048 \ + -nodes \ + -keyout server.key \ + -out server.csr \ + -subj "/OU=TestServer/CN=*" + + openssl x509 \ + -req \ + -days 3650 \ + -sha256 \ + -in server.csr \ + -CA ca.crt \ + -CAkey ca.key \ + -CAcreateserial \ + -out server.cert \ + -extfile <(echo subjectAltName = DNS:localhost) + + openssl req \ + -newkey rsa:2048 \ + -nodes \ + -keyout client.key \ + -out client.csr \ + -subj "/OU=TestClient/CN=*" + + openssl x509 \ + -req \ + -days 3650 \ + -sha256 \ + -in client.csr \ + -CA ca.crt \ + -CAkey ca.key \ + -CAcreateserial \ + -out client.cert + popd +} + function setup_suite { + + write_certs + if [ "$PRIVILEGE_LEVEL" = "priv" ]; then return fi From 7b0c850cd837d08adc20d9d486bd7e50f9e7d36a Mon Sep 17 00:00:00 2001 From: Michael McCracken Date: Thu, 18 Sep 2025 10:27:11 -0700 Subject: [PATCH 3/4] fix: use host and path for credentials search To support different credentials for different paths on a host (e.g. an artifactory server with multiple repositories), we need stacker to send the full path to GetCredentials. GetCredentials searches for the full path, then iterates over subpaths by removing one path component at a time, so the creds from longest matching subpath are returned. Updates the import-http-auth test to show this behavior. Signed-off-by: Michael McCracken --- pkg/stacker/network.go | 8 +++++--- test/import-http-auth.bats | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/stacker/network.go b/pkg/stacker/network.go index cc9c64c09..5737dcac0 100644 --- a/pkg/stacker/network.go +++ b/pkg/stacker/network.go @@ -1,6 +1,7 @@ package stacker import ( + "fmt" "io" "io/fs" "net/http" @@ -85,12 +86,13 @@ func Download(cacheDir string, remoteUrl string, progress bool, expectedHash, re if err != nil { return "", err } - - creds, err := config.GetCredentials(&types.SystemContext{}, u.Hostname()) + key := fmt.Sprintf("%s%s", u.Host, u.Path) + log.Infof("searching creds for key %q", key) + creds, err := config.GetCredentials(&types.SystemContext{}, key) if err != nil { log.Infof("credentials not found for host %s - reason:%s continuing without creds", u.Host, err) } - + log.Infof("found creds for key %q: %+v", key, creds) request.SetBasicAuth(creds.Username, creds.Password) client := &http.Client{} diff --git a/test/import-http-auth.bats b/test/import-http-auth.bats index c207b2ed5..608c7744a 100644 --- a/test/import-http-auth.bats +++ b/test/import-http-auth.bats @@ -8,13 +8,13 @@ img: type: oci url: ${{BUSYBOX_OCI}} imports: - - path: http://localhost:9999/importme + - path: http://localhost:9999/path/to/importme run: | cp /stacker/imports/importme /importme EOF - mkdir -p http_root - echo "please" > http_root/importme + mkdir -p http_root/path/to/ + echo "please" > http_root/path/to/importme wget --quiet https://github.com/m3ng9i/ran/releases/download/v0.1.6/ran_linux_amd64.zip unzip ran_linux_amd64.zip @@ -35,10 +35,19 @@ function teardown() { export XDG_RUNTIME_DIR=$TEST_TMPDIR mkdir -p $TEST_TMPDIR/containers/ + # include valid but wrong creds for the bare hostname and host:port, and + # correct creds for a subpath + # + # NOTE: it is important that the key for each auth dict does NOT end in `/`. + # + # Due to the way that containers/image trims path components, it will never + # search for a subpath with the slash at the end. cat > $TEST_TMPDIR/containers/auth.json < Date: Thu, 18 Sep 2025 13:52:48 -0700 Subject: [PATCH 4/4] docs: update and consolidate credential info Signed-off-by: Michael McCracken --- doc/stacker_yaml.md | 55 +++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/doc/stacker_yaml.md b/doc/stacker_yaml.md index 351baa967..f2e456f7d 100644 --- a/doc/stacker_yaml.md +++ b/doc/stacker_yaml.md @@ -40,10 +40,12 @@ Some directives are irrelevant depending on the type. Supported types are: - `url` is required -- `insecure` is optional. +- `insecure` is optional. If unspecified, the default is is `false`. When `insecure` is specified, stacker attempts to connect via http instead of -https to the Docker Hub. +https to the image registry in the `url`. + +When `insecure` is false, Stacker can authenticate to the image registry using HTTP Basic Authentication. See the section "Credential Handling" for further details. #### `type: tar` @@ -86,21 +88,8 @@ will NOT update this file unless the cache is cleared, to avoid excess network usage. That means that updates after the first time stacker downloads the file will not be reflected. To force re-downloading, use `stacker build --no-cache`. -Stacker supports Basic Authentication for imports from an HTTP server. It will -attempt to find credentials matching the hostname of the URL in the `auth.json` -file documented at -[containers-auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md) - -So for example, this `auth.json` file will allow authentication to example.com -with the username and password of `aw:yeah` (encoded as base64). +Stacker supports Basic Authentication for imports from an HTTPS server. See the section "Credential handling" for details. -```json -{ - "auths": { - "example.com": "YXc6eWVhaA==" - } -} -``` stacker://$name/path/to/file @@ -300,7 +289,7 @@ defaults to the host operating system if not specified. built for, for example, `amd64`, `arm64`, etc. It is an optional field and it defaults to the host machine architecture if not specified. -### Substitution Syntax +## Substitution Syntax Before the yaml is parsed, stacker performs substitution on placeholders in the file of the format `${{VAR}}` or `${{VAR:default}}`. See [Substitution @@ -355,3 +344,35 @@ available for reference: my-build: run: echo "Your layer is ${STACKER_LAYER_NAME}" ``` + +## Credential Handling + +Stacker uses OCI standard credential storage for authenticating to both container registries (for `from` images) and https servers (for `import`s). + +Stacker will look for credentials in the `auth.json` file documented at +[containers-auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md). +The longest matching substring of the import URL is used. + +So for example, this `auth.json` file will allow authentication to example.com +with the username and password of `aw:yeah` (encoded as base64). For a URL +`https://example.com:8080/reg1/p1/path/to/file.txt` or any other file under +`reg1/p1` on that host, this file specifies the alternate creds `arr:narr`: + +```json +{ + "auths": { + "example.com": "YXc6eWVhaA==", + "example.com:8080/reg1/p1": "YXJyOm5hcnI=" + } +} +``` + +NOTE: due to the way the subpaths are broken up to search for the longest +matching subpath, the key in `auth.json` must not have a trailing slash. +`reg1/p1/` will not match any file's path.` + +### Generating auth.json + +`auth.json` is simple enough to generate by hand, but it can also be created and +updated by running `skopeo login $registry_url` for OCI compatible container +image registries.