diff --git a/Makefile b/Makefile index 4a23a502..c409c317 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ BATS = $(TOOLS_D)/bin/bats BATS_VERSION := v1.10.0 # OCI registry ZOT := $(TOOLS_D)/bin/zot -ZOT_VERSION := v2.1.0 +ZOT_VERSION := v2.1.8 UMOCI := $(TOOLS_D)/bin/umoci UMOCI_VERSION := main @@ -124,7 +124,7 @@ go-test: go tool cover -html coverage.txt -o $(HACK_D)/coverage.html .PHONY: download-tools -download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) +download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) $(BATS) $(UMOCI) $(SKOPEO) $(GOLANGCI_LINT): @mkdir -p $(dir $@) diff --git a/doc/stacker_yaml.md b/doc/stacker_yaml.md index 351baa96..f2e456f7 100644 --- a/doc/stacker_yaml.md +++ b/doc/stacker_yaml.md @@ -40,10 +40,12 @@ Some directives are irrelevant depending on the type. Supported types are: - `url` is required -- `insecure` is optional. +- `insecure` is optional. If unspecified, the default is is `false`. When `insecure` is specified, stacker attempts to connect via http instead of -https to the Docker Hub. +https to the image registry in the `url`. + +When `insecure` is false, Stacker can authenticate to the image registry using HTTP Basic Authentication. See the section "Credential Handling" for further details. #### `type: tar` @@ -86,21 +88,8 @@ will NOT update this file unless the cache is cleared, to avoid excess network usage. That means that updates after the first time stacker downloads the file will not be reflected. To force re-downloading, use `stacker build --no-cache`. -Stacker supports Basic Authentication for imports from an HTTP server. It will -attempt to find credentials matching the hostname of the URL in the `auth.json` -file documented at -[containers-auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md) - -So for example, this `auth.json` file will allow authentication to example.com -with the username and password of `aw:yeah` (encoded as base64). +Stacker supports Basic Authentication for imports from an HTTPS server. See the section "Credential handling" for details. -```json -{ - "auths": { - "example.com": "YXc6eWVhaA==" - } -} -``` stacker://$name/path/to/file @@ -300,7 +289,7 @@ defaults to the host operating system if not specified. built for, for example, `amd64`, `arm64`, etc. It is an optional field and it defaults to the host machine architecture if not specified. -### Substitution Syntax +## Substitution Syntax Before the yaml is parsed, stacker performs substitution on placeholders in the file of the format `${{VAR}}` or `${{VAR:default}}`. See [Substitution @@ -355,3 +344,35 @@ available for reference: my-build: run: echo "Your layer is ${STACKER_LAYER_NAME}" ``` + +## Credential Handling + +Stacker uses OCI standard credential storage for authenticating to both container registries (for `from` images) and https servers (for `import`s). + +Stacker will look for credentials in the `auth.json` file documented at +[containers-auth.json](https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md). +The longest matching substring of the import URL is used. + +So for example, this `auth.json` file will allow authentication to example.com +with the username and password of `aw:yeah` (encoded as base64). For a URL +`https://example.com:8080/reg1/p1/path/to/file.txt` or any other file under +`reg1/p1` on that host, this file specifies the alternate creds `arr:narr`: + +```json +{ + "auths": { + "example.com": "YXc6eWVhaA==", + "example.com:8080/reg1/p1": "YXJyOm5hcnI=" + } +} +``` + +NOTE: due to the way the subpaths are broken up to search for the longest +matching subpath, the key in `auth.json` must not have a trailing slash. +`reg1/p1/` will not match any file's path.` + +### Generating auth.json + +`auth.json` is simple enough to generate by hand, but it can also be created and +updated by running `skopeo login $registry_url` for OCI compatible container +image registries. diff --git a/pkg/stacker/network.go b/pkg/stacker/network.go index cc9c64c0..5737dcac 100644 --- a/pkg/stacker/network.go +++ b/pkg/stacker/network.go @@ -1,6 +1,7 @@ package stacker import ( + "fmt" "io" "io/fs" "net/http" @@ -85,12 +86,13 @@ func Download(cacheDir string, remoteUrl string, progress bool, expectedHash, re if err != nil { return "", err } - - creds, err := config.GetCredentials(&types.SystemContext{}, u.Hostname()) + key := fmt.Sprintf("%s%s", u.Host, u.Path) + log.Infof("searching creds for key %q", key) + creds, err := config.GetCredentials(&types.SystemContext{}, key) if err != nil { log.Infof("credentials not found for host %s - reason:%s continuing without creds", u.Host, err) } - + log.Infof("found creds for key %q: %+v", key, creds) request.SetBasicAuth(creds.Username, creds.Password) client := &http.Client{} diff --git a/test/basic-auth.bats b/test/basic-auth.bats new file mode 100644 index 00000000..d31d97e6 --- /dev/null +++ b/test/basic-auth.bats @@ -0,0 +1,36 @@ +load helpers + +function setup() { + stacker_setup + zot_setup_auth + cat > stacker.yaml < $TEST_TMPDIR/containers/auth.json < stacker.yaml <&3 + return 0 fi cat > stacker.yaml <&3 +function write_plain_zot_config { cat > $TEST_TMPDIR/zot-config.json << EOF { "distSpecVersion": "1.1.0-dev", @@ -192,23 +191,97 @@ function zot_setup { "port": "$ZOT_PORT" }, "log": { - "level": "error" + "level": "debug", + "output": "$TEST_TMPDIR/zot.log" } } EOF - # start as a background task + +} + +function write_auth_zot_config { + + htpasswd -Bbn iam careful >> $TEST_TMPDIR/htpasswd + + cat > $TEST_TMPDIR/zot-config.json << EOF +{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "$TEST_TMPDIR/zot", + "gc": true, + "dedupe": true + }, + "http": { + "tls": { + "cert": "$BATS_SUITE_TMPDIR/server.cert", + "key": "$BATS_SUITE_TMPDIR/server.key" + }, + "address": "$ZOT_HOST", + "port": "$ZOT_PORT", + "auth": { + "htpasswd": { + "path": "$TEST_TMPDIR/htpasswd" + } + }, + "accessControl": { + "repositories": { + "**": { + "policies": [{ + "users": [ "iam" ], + "actions": [ "read", "create", "update" ] + }] + } + } + } + }, + "log": { + "level": "debug", + "output": "$TEST_TMPDIR/zot.log", + "audit": "$TEST_TMPDIR/zot-audit.log" + } +} +EOF + +} + +function zot_setup { + write_plain_zot_config + start_zot +} + +function zot_setup_auth { + write_auth_zot_config + start_zot USE_TLS +} + +function start_zot { + ZOT_USE_TLS=$1 + echo "# starting zot at $ZOT_HOST:$ZOT_PORT" >&3 + # start as a background task + zot verify $TEST_TMPDIR/zot-config.json zot serve $TEST_TMPDIR/zot-config.json & pid=$! + + echo "zot is running at pid $pid" + cat $TEST_TMPDIR/zot.log # wait until service is up count=5 up=0 + while [[ $count -gt 0 ]]; do if [ ! -d /proc/$pid ]; then echo "zot failed to start or died" exit 1 fi up=1 - curl -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 + if [[ -n $ZOT_USE_TLS ]]; then + echo "testing zot at https://$ZOT_HOST:$ZOT_PORT" + curl -v --cacert $BATS_SUITE_TMPDIR/ca.crt -u "iam:careful" -f https://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 + else + echo "testing zot at http://$ZOT_HOST:$ZOT_PORT" + curl -v -f http://$ZOT_HOST:$ZOT_PORT/v2/ || up=0 + fi + if [ $up -eq 1 ]; then break; fi sleep 1 count=$((count - 1)) @@ -217,8 +290,15 @@ EOF echo "Timed out waiting for zot" exit 1 fi + + echo "# zot is up" >&3 # setup a OCI client - regctl registry set --tls=disabled $ZOT_HOST:$ZOT_PORT + if [[ -n $ZOT_USE_TLS ]]; then + regctl registry set $ZOT_HOST:$ZOT_PORT + else + regctl registry set --tls=disabled $ZOT_HOST:$ZOT_PORT + fi + } function zot_teardown { diff --git a/test/import-http-auth.bats b/test/import-http-auth.bats index c207b2ed..608c7744 100644 --- a/test/import-http-auth.bats +++ b/test/import-http-auth.bats @@ -8,13 +8,13 @@ img: type: oci url: ${{BUSYBOX_OCI}} imports: - - path: http://localhost:9999/importme + - path: http://localhost:9999/path/to/importme run: | cp /stacker/imports/importme /importme EOF - mkdir -p http_root - echo "please" > http_root/importme + mkdir -p http_root/path/to/ + echo "please" > http_root/path/to/importme wget --quiet https://github.com/m3ng9i/ran/releases/download/v0.1.6/ran_linux_amd64.zip unzip ran_linux_amd64.zip @@ -35,10 +35,19 @@ function teardown() { export XDG_RUNTIME_DIR=$TEST_TMPDIR mkdir -p $TEST_TMPDIR/containers/ + # include valid but wrong creds for the bare hostname and host:port, and + # correct creds for a subpath + # + # NOTE: it is important that the key for each auth dict does NOT end in `/`. + # + # Due to the way that containers/image trims path components, it will never + # search for a subpath with the slash at the end. cat > $TEST_TMPDIR/containers/auth.json <