Restore the database stack test

.github/workflows/test-database.yml

@@ -0,0 +1,33 @@
+name: Database Stack Test
+
+on:
+  pull_request:
+    branches:
+      - '*'
+  push:
+    branches:
+      - '*'
+
+# Note: this test depends on having kind installed in the runner environment.
+# It turns out that kind is pre-installed in the ubuntu-latest. It may be worthwhile
+# checking which version is installed in the event that unexplained test failures occur.
+
+jobs:
+  test:
+    name: "Run k8s deploy test suite"
+    runs-on: ubuntu-24.04
+    steps:
+      - name: "Clone project repository"
+        uses: actions/checkout@v3
+      - name: "Install uv"
+        uses: astral-sh/setup-uv@v4
+        with:
+          python-version: '3.12'
+      - name: "Print kind version"
+        run: kind version
+      - name: "Generate build version file"
+        run: ./scripts/create_build_tag_file.sh
+      - name: "Build local shiv package"
+        run: ./scripts/build_shiv_package.sh
+      - name: "Run k8s deploy tests"
+        run: ./tests/database/run-test.sh

src/stack/repos/repo_util.py

@@ -276,7 +276,9 @@ def clone_all_repos_for_stack(stack, include=None, exclude=None, pull=False, git
             os.makedirs(dev_root_path)
 
         repos_in_scope = [req_stack.get_repo_ref()]
-        repos_in_scope.extend(req_stack.get("repos", []))
+        stack_repos = req_stack.get("repos", [])
+        if stack_repos is not None:
+            repos_in_scope.extend(req_stack.get("repos", []))
 
         # containers can reference an external repo
         containers_in_scope = build_util.get_containers_in_scope(req_stack)

tests/database/run-test.sh

@@ -7,25 +7,28 @@ if [ -n "$STACK_SCRIPT_DEBUG" ]; then
     env
 fi
 
+if ! command -v kind &> /dev/null; then
+    echo "Error: 'kind' is not installed or not available on the PATH"
+    exit 1
+fi
+
 if [ "$1" == "from-path" ]; then
-    TEST_TARGET_SO="stack"
+    TEST_TARGET_STACK="stack"
 else
-    TEST_TARGET_SO=$( ls -t1 ./package/stack* | head -1 )
+    TEST_TARGET_STACK=$( ls -t1 ./package/stack* | head -1 )
 fi
 
-export STACK_USE_BUILTIN_STACK=true
-
-stack="test-database"
+stack="test-database-stack"
 spec_file=${stack}-spec.yml
 deployment_dir=${stack}-deployment
 
 # Helper functions: TODO move into a separate file
 wait_for_pods_started () {
     for i in {1..50}
     do
-        local ps_output=$( $TEST_TARGET_SO manage --dir $test_deployment_dir ps )
+        local ps_output=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir ps )
 
-        if [[ "$ps_output" == *"Running containers:"* ]]; then
+        if [[ "$ps_output" == *"id:"* ]]; then
             # if ready, return
             return
         else
@@ -42,7 +45,7 @@ wait_for_test_complete () {
     for i in {1..50}
     do
 
-        local log_output=$( $TEST_TARGET_SO manage --dir $test_deployment_dir logs )
+        local log_output=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir logs )
 
     if [[ "${log_output}" == *"Database test client: test complete"* ]]; then
             # if ready, return
@@ -59,26 +62,41 @@ wait_for_test_complete () {
 
 
 delete_cluster_exit () {
-    $TEST_TARGET_SO manage --dir $test_deployment_dir stop --delete-volumes
+    $TEST_TARGET_STACK manage --dir $test_deployment_dir stop --delete-volumes
     exit 1
 }
 
+# We make a directory within which our test will create files
+STACK_TEST_DIR=~/stack-test/database-test-dir
 # Set a non-default repo dir
-export STACK_REPO_BASE_DIR=~/stack-test/repo-base-dir
-echo "Testing this package: $TEST_TARGET_SO"
+export STACK_REPO_BASE_DIR=${STACK_TEST_DIR}/repo-base-dir
+echo "Testing this package: $TEST_TARGET_STACK"
 echo "Test version command"
-reported_version_string=$( $TEST_TARGET_SO version )
+reported_version_string=$( $TEST_TARGET_STACK version )
 echo "Version reported is: ${reported_version_string}"
-echo "Cloning repositories into: $STACK_REPO_BASE_DIR"
-rm -rf $STACK_REPO_BASE_DIR
+echo "Using test directory: $STACK_TEST_DIR"
+rm -rf $STACK_TEST_DIR
+mkdir -p $STACK_TEST_DIR
 mkdir -p $STACK_REPO_BASE_DIR
-$TEST_TARGET_SO fetch repositories --stack ${stack}
-$TEST_TARGET_SO build containers --stack ${stack}
+# We must delete any instances of the test-container in the local registory
+# otherwise we'll skip building it below
+existing_test_images=$(docker image ls -q --filter=reference=bozemanpass/test-database-client | uniq)
+if [ -n "$existing_test_images" ]; then
+  docker image rm -f ${existing_test_images}
+fi
+existing_test_images=$(docker image ls -q --filter=reference=bozemanpass/test-database-container | uniq)
+if [ -n "$existing_test_images" ]; then
+  docker image rm -f ${existing_test_images}
+fi
+# Fetch the test stacks
+echo "Fetching test stack repo into: $STACK_REPO_BASE_DIR"
+$TEST_TARGET_STACK fetch repo github.com/bozemanpass/stack-test-stacks
+$TEST_TARGET_STACK prepare --stack ${stack}
 # Test basic stack deploy to k8s
 test_deployment_dir=$STACK_REPO_BASE_DIR/${deployment_dir}
 test_deployment_spec=$STACK_REPO_BASE_DIR/${spec_file}
 
-$TEST_TARGET_SO --stack ${stack} deploy --deploy-to k8s-kind init --output $test_deployment_spec
+$TEST_TARGET_STACK init --stack ${stack} --deploy-to k8s-kind --output $test_deployment_spec
 # Check the file now exists
 if [ ! -f "$test_deployment_spec" ]; then
     echo "deploy init test: spec file not present"
@@ -90,7 +108,7 @@ echo "deploy init test: passed"
 # Switch to a full path for the data dir so it gets provisioned as a host bind mounted volume and preserved beyond cluster lifetime
 sed -i "s|^\(\s*db-data:$\)$|\1 ${test_deployment_dir}/data/db-data|" $test_deployment_spec
 
-$TEST_TARGET_SO --stack ${stack} deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
+$TEST_TARGET_STACK deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir
 # Check the deployment dir exists
 if [ ! -d "$test_deployment_dir" ]; then
     echo "deploy create test: deployment directory not present"
@@ -100,11 +118,11 @@ fi
 echo "deploy create test: passed"
 
 # Try to start the deployment
-$TEST_TARGET_SO manage --dir $test_deployment_dir start
+$TEST_TARGET_STACK manage --dir $test_deployment_dir start
 wait_for_pods_started
 # Check logs command works
 wait_for_test_complete
-log_output_1=$( $TEST_TARGET_SO manage --dir $test_deployment_dir logs )
+log_output_1=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir logs )
 if [[ "$log_output_1" == *"Database test client: test data does not exist"* ]]; then
     echo "Create database content test: passed"
 else
@@ -113,14 +131,14 @@ else
 fi
 
 # Stop then start again and check the volume was preserved
-$TEST_TARGET_SO manage --dir $test_deployment_dir stop
+$TEST_TARGET_STACK manage --dir $test_deployment_dir stop
 # Sleep a bit just in case
 sleep 20
-$TEST_TARGET_SO manage --dir $test_deployment_dir start
+$TEST_TARGET_STACK manage --dir $test_deployment_dir start
 wait_for_pods_started
 wait_for_test_complete
 
-log_output_2=$( $TEST_TARGET_SO manage --dir $test_deployment_dir logs )
+log_output_2=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir logs )
 if [[ "$log_output_2" == *"Database test client: test data already exists"* ]]; then
     echo "Retain database content test: passed"
 else
@@ -129,5 +147,5 @@ else
 fi
 
 # Stop and clean up
-$TEST_TARGET_SO manage --dir $test_deployment_dir stop --delete-volumes
+$TEST_TARGET_STACK manage --dir $test_deployment_dir stop --delete-volumes
 echo "Test passed"

tests/smoke-test/run-smoke-test.sh

@@ -19,7 +19,6 @@ echo "Test version command"
 reported_version_string=$( $TEST_TARGET_STACK version )
 echo "Version reported is: ${reported_version_string}"
 echo "Using test directory: $STACK_TEST_DIR"
-echo "Cloning repositories into: $STACK_REPO_BASE_DIR"
 rm -rf $STACK_TEST_DIR
 mkdir -p $STACK_TEST_DIR
 mkdir -p $STACK_REPO_BASE_DIR
@@ -30,6 +29,7 @@ if [ -n "$existing_test_images" ]; then
   docker image rm -f ${existing_test_images}
 fi
 # Fetch the test stacks
+echo "Fetching test stac repo into: $STACK_REPO_BASE_DIR"
 $TEST_TARGET_STACK fetch repo github.com/bozemanpass/stack-test-stacks
 # Test building the a stack container
 $TEST_TARGET_STACK prepare --stack test