Integration test a local Sunlight log

.github/workflows/integration.yml

@@ -0,0 +1,9 @@
+name: Sunlight Integration Test
+on: [push, pull_request]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: ./test.sh
+        working-directory: ./integration/

integration/Dockerfile

@@ -0,0 +1,26 @@
+ARG GO_VERSION
+
+FROM golang:${GO_VERSION}-bookworm as build
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY cmd ./cmd
+COPY internal ./internal
+COPY *.go ./
+
+WORKDIR /app/cmd/sunlight
+RUN go build -o /bin/sunlight .
+
+FROM debian:bookworm
+
+RUN apt-get -y update &&  \
+     apt-get install --no-install-recommends  \
+     -y curl ca-certificates && \
+     rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /bin/sunlight /bin/sunlight
+
+CMD /bin/sunlight

integration/aws-setup.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -xeuo pipefail
+
+# This script conditionally creates an S3 bucket and DynamoDB table for
+# the sunlight integration tests to use
+
+export AWS_REGION=localhost
+export AWS_ACCESS_KEY_ID=minioadmin
+export AWS_SECRET_ACCESS_KEY=minioadmin
+
+aws s3api head-bucket --bucket sunlight-integration --endpoint-url http://minio:9000 || \
+  aws s3api create-bucket --bucket sunlight-integration --endpoint-url http://minio:9000
+
+aws dynamodb describe-table --table-name sunlight-integration --endpoint-url http://dynamo:8000 || \
+  aws dynamodb create-table --endpoint-url http://dynamo:8000 \
+    --table-name sunlight-integration \
+    --attribute-definitions AttributeName=logID,AttributeType=B \
+    --key-schema AttributeName=logID,KeyType=HASH \
+    --provisioned-throughput ReadCapacityUnits=1,WriteCapacityUnits=1

integration/docker-compose.yml

@@ -0,0 +1,85 @@
+services:
+  minio:
+    image: "quay.io/minio/minio:RELEASE.2024-08-03T04-33-23Z"
+    command: ["server", "--address", ":9000", "--console-address", ":9001", "/data"]
+    ports:
+      - "127.0.0.1:9000:9000"
+      - "127.0.0.1:9001:9001"
+    volumes:
+      - minio_data:/data
+    environment:
+      - MINIO_DOMAIN=minio
+    networks:
+      net:
+        aliases:
+          - sunlight-integration.minio
+    healthcheck:
+      test: [ "CMD", "mc", "ready", "local" ]
+      interval: 5s
+  dynamo:
+    command: "-jar DynamoDBLocal.jar -sharedDb -dbPath /data"
+    image: "public.ecr.aws/aws-dynamodb-local/aws-dynamodb-local:2.5.2"
+    working_dir: /home/dynamodblocal
+    ports:
+      - "127.0.0.1:8000:8000"
+    user: root
+    volumes:
+      - dynamo_data:/data
+    networks:
+      net:
+    healthcheck:
+      test: ["CMD", "curl", "-I", "http://localhost:8000"]
+      interval: 5s
+  aws-setup:
+    image: "public.ecr.aws/aws-cli/aws-cli:2.17.23"
+    entrypoint: /bin/sh
+    command: /bin/aws-setup.sh
+    volumes:
+      - type: bind
+        source: ./aws-setup.sh
+        target: /bin/aws-setup.sh
+    depends_on:
+      minio:
+        condition: service_healthy
+      dynamo:
+        condition: service_healthy
+    networks:
+      net:
+  sunlight:
+    build:
+      dockerfile: integration/Dockerfile
+      context: ../
+      args:
+        GO_VERSION: 1.22.5
+    environment:
+      - AWS_ACCESS_KEY_ID=minioadmin
+      - AWS_SECRET_ACCESS_KEY=minioadmin
+    networks:
+      net:
+    ports:
+      - "127.0.0.1:7600:7600"
+    working_dir: /
+    command: ["/bin/sunlight", "-testcert", "-c", "/etc/sunlight/integration-test-config.yaml"]
+    depends_on:
+      aws-setup:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "curl", "-I", "https://sunlight:7600", "--cacert", "/etc/sunlight/roots.pem"]
+      interval: 5s
+    volumes:
+      - cache:/cache
+      - type: bind
+        source: ./etc/sunlight
+        target: /etc/sunlight
+      - type: bind
+        source: ./sunlight.pem
+        target: /sunlight.pem
+      - type: bind
+        source: ./sunlight-key.pem
+        target: /sunlight-key.pem
+volumes:
+  minio_data:
+  dynamo_data:
+  cache:
+networks:
+  net:

integration/etc/sunlight/ci-log-1-seed.bin

@@ -0,0 +1 @@
+>Ś1�b黙u"���(�¾���_z�

integration/etc/sunlight/integration-test-config.yaml

@@ -0,0 +1,22 @@
+listen: ":7600"
+
+dynamodb:
+  region: "localhost"
+  table: "sunlight-integration"
+  endpoint: "http://dynamo:8000/"
+
+logs:
+  - name: sunlight/ci-log-1
+    shortname: ci-log-1
+    inception: 2024-08-06 # TODO - need to template today's date in here
+    roots: /etc/sunlight/roots.pem
+    seed: /etc/sunlight/ci-log-1-seed.bin
+    notafterstart: 2000-01-01T00:00:00Z
+    notafterlimit: 2100-01-01T00:00:00Z
+    s3bucket: sunlight-integration
+    s3endpoint: http://minio:9000
+    s3region: localhost
+    s3keyprefix: "log1/"
+    poolsize: 750
+    cache: /cache/ci-log-1.db
+    publickey: "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAElTZfG/A38FkaRbLUG+llfR+eK6YxxUk2OMqtptnIeIeRdVI+RoTuUcCquhUskv8U5r4KSuCRFsCiYSN/V+WvDQ=="

integration/etc/sunlight/roots.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSzCCAjOgAwIBAgIIALwoFRjyIKswDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
+AxMVbWluaWNhIHJvb3QgY2EgYmMyODE1MCAXDTI0MDgwNTIxMjcwNVoYDzIxMjQw
+ODA1MjEyNzA1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSBiYzI4MTUwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAm6dM8oifd5Dz632aoX0ABojS
+tg5qujmDa63j9WF1Vbw7AYX1naSXvSVKFqaZ+DJKjy75Nl2+vTMLOhZ+Pbe7BBcL
+c4ctf5lKrUt7qs0IPML32CG/KIAXe67ciGGbgGIJdduUQm5vLndpsq+JTYgCJp3D
+cejruQNSK4EtClrCamLQRl9Zyf4IDmEMtiGaKD6s/uB1QLFOh5BuW0KnWh4SnX1q
+ogS3pXsVeY32AzJ0bVqAn1JDSZ919kEO15Di8BlXuaLsRdGfDnaYcIuJNW7oOMUS
+a1WQUY15vzuTBp8878al50Af7hJPeI/V6xjVc6XGZTqd7aRHNEyHHEsVZS8/AgMB
+AAGjgYYwgYMwDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggr
+BgEFBQcDAjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBRn7Efh3EPXY3EB
+gtCtuAT8dZYCOzAfBgNVHSMEGDAWgBRn7Efh3EPXY3EBgtCtuAT8dZYCOzANBgkq
+hkiG9w0BAQsFAAOCAQEAOClgPWvmfKcZIfqSZ2PODN93V/emeAv8uYYqml+mBvhB
+chNk2gmfxI39D1JkHsiOSlYFo7UNe4tYoJ3af4xR+uQwD1bkbeK9ZLr6lQbUBOpw
+KD1T7pNsjk/l0YEoj/TqrutG7Eb2QCPDH6kyoPTpEmZQ4lLZLtVc/a7eW6ekH6j6
+HrPZWDEpiuqgIO61ftVZk8HWlRFKyiki1Yx+SqLpGhKnhCwn/fpVaPTaPuWuC2jv
+6NsJMd1XWCKJ4D6Q0i/yvuQ/3WIhZLMm2fsxAX04NBcK3lpuwHAsvfOEK65h6KeT
+CTlMrTwqBw6l0U+gzYHbH8XBx/zL9kNB/QAB6l1rOw==
+-----END CERTIFICATE-----

integration/integration_test.go

@@ -0,0 +1,18 @@
+//go:build integration
+
+// This is an integration test suite for Sunlight.
+//
+// It requires a running Sunlight instance to test against, which can be run from
+// the included `docker-compose.yml` configuration via `docker compose up` prior
+// to running
+package integration
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestPlaceholder
+func TestPlaceholder(t *testing.T) {
+	fmt.Println("it works!")
+}

integration/sunlight-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA5iVqZUoMdEE+rO7rGB6jdPKtcDOw4hcbgJzhtSmJp1sD+jpL
+2Zs/Wd0YgHYHgJm0q9Q4teJ2JUpQy4ON0MQJ2ZuSaMeUF7egMvnKrogX1EFtPFZV
+VCa90jjCSaG44Jt8+Y2wvtRFEiE+zT5bpnU3gA3Xb/nCvOG8G9S4+DDnPrSfg/m2
+oITwAnoxX/mXkHskndjenRIC6MUsxuAD4nBvz8H9oPoweD8eSxylbTCOZuVEZ+61
+8tdxytJZ2tg4EMbbb5oAJAcGgFUVfB6zmsArvS2i23aHSbGuFyK3/cKtmEyNAIWV
+0O1PIylPN8626qm0ql2h1R7V8h0yDzRTID+OtwIDAQABAoIBADBHYQ5mfj5VAYTd
+icvBgaCi1+dVsKF9H3PXdTG/yPSmjysK16aKfrunOeCY4fIETHZPSqY9H3OrJ0/K
+BRHqTfasQAZi95oSFzUaWX5LZlRMdVHW+odwqRlUds1jc5kKhSkEpDMj7GllN/tu
+PWOp6z6dqLgAuO1OnVtpQHQwLhJlA75MaBZvLjnVJTN9n8N0e+G3gK58ao1k2flV
+ZGca6NPSfT74weZANT0B9gfWXUC7m5tNQ41mmOTBrPWSJ6fm9y5/MuOrqBWt58Ii
+kAJDFW0Fl84nmcon4mowWXJ8rk3GolUF0YWzrpIPR1DywZKcG4lKoypVXnUDL89j
+vHA9KIECgYEA61B7ktSP4jptC8lEauHPXpWtL0svoGLt2gs2fv1Fl1uPiuFIimh/
+d/pI4+FhQ11YQUy1QHdgIWTfQ4c/48tABA0KtKGMpto8TEosrwxk0pMmnq1OQjMY
+2GetYL3ochKRL53PzkjLr6UpzFJHQlXJiyNSXnUx+yvwTTKaIq6WpqcCgYEA+mCg
+gb33zCR/Rg07oZn1v4LINGKMVkYYIrmWpHeKVa8zZYm7BHNeTyW3Ujv0vPCCIZcm
+HVGQ99ls/B9S9k88OdM1hDZ1FwChvALBCswe35rjZdU8KHQxEYgUXrgto63cbvhY
+QVJB3owdk7nEyGyynOBiQGCV6uSIt4cahxGe6XECgYA7R7U50i7FMS7Tb0q7GlPh
+YheU+H3xC7sb2QNLjoZ/kONie0qJA7ZJBVZjgh28mMFpAegPa4NeJklQe7wOqN9h
+pd9yjdgfGI9kMLN9DcRAgC2Uf9hdOP4jVksJOg5a3m6Q7lKxhwk5IISdIoUtIut5
+D9bqUN/chL8u03adEhqFtQKBgQDRo/qA5+kF20FeDFJzuw/TbNPpWOKcklA/Mxn5
+V8ZSfEISsMIQtfCvYKsgheckFFVBfueBAPzbKghhCnFSn4GUAU78BwT/X54MaFnu
+in8HRw9EAsk+LhuvzDsSHDDKrha2K+bputBAYR2axZ8JGkprpE1gDf85EGWRFeSW
+V6CxYQKBgQDXcqqNBVdx3aAWIT+GRFBtdn9XrQ9vcwqQgaxHrvnaIk/eBCwcMpph
+H/6OAYxfqIlVfQa+6HRpfpyfHh4j5mKuFyOUsfqvpp5cI6xkweUlTFRaOPc9riAO
+Ca9UM10Gcw2e6xcFlanQBAOXdgxUI7SWAE3iFclLh43GrDjtfAB9Bg==
+-----END RSA PRIVATE KEY-----

integration/sunlight.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDKjCCAhKgAwIBAgIINkivnOllZpMwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
+AxMVbWluaWNhIHJvb3QgY2EgYmMyODE1MB4XDTI0MDgwNTIxMjcwNVoXDTI2MDkw
+NDIxMjcwNVowEzERMA8GA1UEAxMIc3VubGlnaHQwggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDmJWplSgx0QT6s7usYHqN08q1wM7DiFxuAnOG1KYmnWwP6
+OkvZmz9Z3RiAdgeAmbSr1Di14nYlSlDLg43QxAnZm5Jox5QXt6Ay+cquiBfUQW08
+VlVUJr3SOMJJobjgm3z5jbC+1EUSIT7NPlumdTeADddv+cK84bwb1Lj4MOc+tJ+D
++baghPACejFf+ZeQeySd2N6dEgLoxSzG4APicG/Pwf2g+jB4Px5LHKVtMI5m5URn
+7rXy13HK0lna2DgQxttvmgAkBwaAVRV8HrOawCu9LaLbdodJsa4XIrf9wq2YTI0A
+hZXQ7U8jKU83zrbqqbSqXaHVHtXyHTIPNFMgP463AgMBAAGjdTBzMA4GA1UdDwEB
+/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/
+BAIwADAfBgNVHSMEGDAWgBRn7Efh3EPXY3EBgtCtuAT8dZYCOzATBgNVHREEDDAK
+gghzdW5saWdodDANBgkqhkiG9w0BAQsFAAOCAQEAq8CS/CRx+m6ZICoPVEMYm71X
+9oUFVlon6erZpWD6P4StDkzoVtjfRtighMPC0ldh44Tat99iHho4itkyN1FNBYCQ
+LgBzIVWceXP9fGgSjJ8o1A/JiMM/c6i0rfM8dP6l32vnCoG8zwaRQX5nlarTTjL2
+eGpqYfB1/HwTCwpEbPpNA30YQ5793bIRACn12FNI/hBA3lGkhpqn0T43tLyYSkXJ
+x/3w8GFQpf3B2r/UnXZ6xUZzBL0U6WrTu17fKk6TkD2ihvvc72CkpVdpol4Jiskp
+l4B0dhwK8b5czAA0wy7Hnvlf1ah5zkbUZJZU0yuFhl4rzOooFL1iQi9qtvuO/A==
+-----END CERTIFICATE-----

integration/test.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -eu
+
+stop() {
+  ret=$?
+
+  # Print the logs from each of the service containers.
+  # This list needs to be kept in sync with ./integration/docker-compose.yml
+  for ctr in minio dynamo aws-setup sunlight; do
+    name="integration-$ctr-1"
+    echo "::group::$name logs"
+    docker logs -t "$name" || echo "no logs"
+    echo "::endgroup::"
+  done
+
+  docker compose down
+
+  exit $ret
+}
+
+trap stop EXIT
+
+# Build containers:
+echo "::group::docker compose build"
+docker compose build
+echo "::endgroup::"
+
+# Start services in background
+echo "::group::docker compose up"
+docker compose up -d --wait
+echo "::endgroup::"
+
+# Run integration tests
+
+echo "::group::Integration Test"
+RET=$(go test -tags=integration)
+if [ "$RET" != "0" ]; then
+  echo "tests returned $RET"
+  exit 1
+fi
+echo "Success"
+echo "::endgroup::"