feat: minikube k8s can pull from ECR (#404)

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
       - image: python:3.9.10
     steps:
       - checkout
-      - run: make dependencies
+      - run: make hokusai
       - run: make test
       - run: coverage report
       - run: make pyinstaller-build-onefile ARTIFACT_LABEL=beta
@@ -32,7 +32,7 @@ jobs:
           command: |
             curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg"
             sudo installer -pkg AWSCLIV2.pkg -target /
-      - run: make dependencies
+      - run: make hokusai
       - run: pyenv rehash
       - run: make test
       - run: coverage report
@@ -87,6 +87,7 @@ jobs:
             done
             kubectl --context staging version
             kubectl --context staging cluster-info
+            kubectl --context staging get pods --all-namespaces
       - run:
           name: start production minikube cluster
           command: |
@@ -106,22 +107,34 @@ jobs:
             done
             kubectl --context production version
             kubectl --context production cluster-info
+            kubectl --context production get pods --all-namespaces
       - run:
-          name: fix RBAC
+          name: log into ECR
           command: |
-            # make default account cluster-admin
-            kubectl create clusterrolebinding add-on-cluster-admin --clusterrole cluster-admin --serviceaccount=kube-system:default
+            aws_account_id=$(aws sts get-caller-identity --query "Account" --output text)
+            aws ecr get-login-password --region "$AWS_DEFAULT_REGION" | \
+              docker login --username AWS \
+                --password-stdin ${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com
       - run:
-          name: dump cluster-info
+          name: setup regcred for staging minikube/k8s
           command: |
-            kubectl cluster-info
-            kubectl get pods --all-namespaces
+            kubectl --context staging create secret generic regcred \
+              --from-file=.dockerconfigjson=${HOME}/.docker/config.json \
+              --type=kubernetes.io/dockerconfigjson
+            kubectl --context staging patch serviceaccount default -p '{"imagePullSecrets": [{"name": "regcred"}]}'
+      - run:
+          name: setup regcred for production minikube/k8s
+          command: |
+            kubectl --context production create secret generic regcred \
+              --from-file=.dockerconfigjson=${HOME}/.docker/config.json \
+              --type=kubernetes.io/dockerconfigjson
+            kubectl --context production patch serviceaccount default -p '{"imagePullSecrets": [{"name": "regcred"}]}'
       - run:
           name: setup hokusai global config file
           command: |
             cp test/integration/fixtures/ci_hokusai_global_config.yml ${HOME}/.hokusai.yml
       - run: pyenv local 3.10.2
-      - run: make dependencies
+      - run: make hokusai
       - run: pyenv rehash
       - run: make integration
       - run: coverage report
@@ -143,7 +156,7 @@ jobs:
       - checkout
       - run: apt-get -qq update
       - run: apt-get -qq install awscli
-      - run: make dependencies
+      - run: make hokusai
       - run: scripts/update_version_file.sh
       - run: make pyinstaller-build-onefile ARTIFACT_LABEL=beta
       - run: make publish-to-s3 ARTIFACT_LABEL=beta
@@ -169,7 +182,7 @@ jobs:
           command: |
             curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg"
             sudo installer -pkg AWSCLIV2.pkg -target /
-      - run: make dependencies
+      - run: make hokusai
       - run: pyenv rehash
       - run: scripts/update_version_file.sh
       - run: make pyinstaller-build-onedir ARTIFACT_LABEL=beta
@@ -244,7 +257,7 @@ jobs:
       - checkout
       - run: apt-get -qq update
       - run: apt-get -qq install awscli
-      - run: make dependencies
+      - run: make hokusai
       - run: scripts/update_version_file.sh
       - run: make pyinstaller-build-onefile ARTIFACT_LABEL=$(cat hokusai/VERSION)
       - run: make pyinstaller-build-onefile ARTIFACT_LABEL=latest
@@ -272,7 +285,7 @@ jobs:
           command: |
             curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg"
             sudo installer -pkg AWSCLIV2.pkg -target /
-      - run: make dependencies
+      - run: make hokusai
       - run: pyenv rehash
       - run: scripts/update_version_file.sh
       - run: make pyinstaller-build-onedir ARTIFACT_LABEL=$(cat hokusai/VERSION)
@@ -309,7 +322,7 @@ jobs:
       - image: python:3.9.10
     steps:
       - checkout
-      - run: make dependencies
+      - run: make hokusai
       - run: scripts/update_version_file.sh
       - run: make publish-to-pip
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: dependencies test integration integration-local pyinstaller-build-onefile pyinstaller-build-onedir publish-to-s3 publish-to-s3-canonical build-docker-image publish-to-dockerhub-beta publish-to-dockerhub-canonical-and-latest publish-to-pip publish-to-github clean
+.PHONY: hokusai test integration integration-local pyinstaller-build-onefile pyinstaller-build-onedir publish-to-s3 publish-to-s3-canonical build-docker-image publish-to-dockerhub-beta publish-to-dockerhub-canonical-and-latest publish-to-pip publish-to-github clean
 
 # a var passed in as an argument to 'make' command moots its ?= assgiment
 AWS ?= $(shell which aws)
@@ -10,7 +10,7 @@ RELEASE_MINOR_VERSION ?= $(shell cat hokusai/VERSION | awk -F"." '{ print $$1"."
 ARTIFACT_LABEL ?= $(shell cat hokusai/VERSION)
 BINARY_SUFFIX ?= -$(ARTIFACT_LABEL)-$(shell uname -s)-$(shell uname -m)
 
-dependencies:
+hokusai:
 	pip install --upgrade pip
 	# pin version due to https://github.com/python-poetry/poetry/issues/7184
 	pip install poetry==1.2.2 --quiet --ignore-installed

scripts/integration_test_local.sh

@@ -1,11 +1,30 @@
-# run integration tests locally
+### run integration tests locally
+
+
+## requirements
+
+# - awscli
+# - coverage
+# - docker
+# - jq
+# - kubectl
+# - minikube
+# - pytest
+
+
+## prepare env
 
 export K8S_VERSION=v1.21.14
 export KUBE_CONFIG_PATH=${HOME}/.kube/config
 export KUBE_CONFIG_BACKUP_PATH=${HOME}/.kube/config.bak
-export BACKED_UP_KUBE_CONFIG=false
+export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text)
+export AWS_REGION=$(aws configure get region)
+export DOCKER_CONFIG_JSON_PATH=${HOME}/.docker/config.json
+
+
+## functions
 
-function start_minikube(){
+function start_minikube() {
   profile=$1
   minikube delete --profile "$profile"
   minikube start --profile "$profile" --kubernetes-version="$K8S_VERSION"
@@ -15,23 +34,82 @@ function start_minikube(){
   done
 }
 
-if ls "$KUBE_CONFIG_PATH"
-then
-  echo "Backing up $KUBE_CONFIG_PATH ..."
-  mv "$KUBE_CONFIG_PATH" "$KUBE_CONFIG_BACKUP_PATH"
-  export BACKED_UP_KUBE_CONFIG=true
-fi
+function create_real_docker_config_json() {
+  # on Mac, Docker config.json points to credsStore
+  # where the creds are really stored,
+  # create a file that has the creds inline
+  local dockercfg=$(mktemp ~/.docker/config.json.XXX)
+  local storetype=$(jq -r .credsStore < $DOCKER_CONFIG_JSON_PATH)
+  local registry=''
+  (
+    echo '{'
+    echo '    "auths": {'
+    for registry in $(docker-credential-$storetype list | jq -r 'to_entries[] | .key'); do
+      if [ ! -z "$FIRST" ]; then
+          echo '        },'
+      fi
+      FIRST='true'
+      credential=$(echo $registry | docker-credential-$storetype get | jq -jr '"\(.Username):\(.Secret)"' | base64)
+      echo '        "'$registry'": {'
+      echo '            "auth": "'$credential'"'
+    done
+    echo '        }'
+    echo '    }'
+    echo '}'
+  ) > $dockercfg
+  echo "$dockercfg"
+}
 
+function back_up_kube_config() {
+  # don't let minikube overwrite user's real k8s kubeconfig
+  local backed_up_kube_config=''
+  if ls "$KUBE_CONFIG_PATH"
+  then
+    echo "Backing up $KUBE_CONFIG_PATH ..."
+    mv "$KUBE_CONFIG_PATH" "$KUBE_CONFIG_BACKUP_PATH"
+    backed_up_kube_config='true'
+  fi
+  echo "$backed_up_kube_config"
+}
+
+function restore_kube_config() {
+  echo "Restoring $KUBE_CONFIG_PATH ..."
+  mv "$KUBE_CONFIG_BACKUP_PATH" "$KUBE_CONFIG_PATH"
+}
+
+function create_regcred() {
+  # create ecr image pull secret in minikube/k8s
+  aws ecr get-login-password --region "$AWS_REGION" | \
+    docker login --username AWS \
+      --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
+  local dockercfg=$(create_real_docker_config_json)
+  # ensure file is deleted when script exits
+  trap "rm -f $dockercfg" EXIT
+  create_regcred_for_env 'staging' $dockercfg
+  create_regcred_for_env 'production' $dockercfg
+  rm -f "$dockercfg"
+}
+
+function create_regcred_for_env() {
+  local environment=$1
+  local dockercfg=$2
+  kubectl --context "$environment" create secret generic regcred \
+    --from-file=.dockerconfigjson="$dockercfg" \
+    --type=kubernetes.io/dockerconfigjson
+  kubectl --context "$environment" patch serviceaccount default -p '{"imagePullSecrets": [{"name": "regcred"}]}'
+}
+
+
+## main
+
+kube_config_is_backed_up=$(back_up_kube_config)
 start_minikube staging
 start_minikube production
-
+create_regcred
 coverage run -m pytest test/integration
-
 minikube stop --profile staging
 minikube stop --profile production
-
-if [ "$BACKED_UP_KUBE_CONFIG" = "true" ]
+if [ "$kube_config_is_backed_up" = "true" ]
 then
-  echo "Restoring $KUBE_CONFIG_PATH ..."
-  mv "$KUBE_CONFIG_BACKUP_PATH" "$KUBE_CONFIG_PATH"
+  restore_kube_config
 fi

test/integration/conftest.py

@@ -9,7 +9,7 @@
 from hokusai.lib.common import ansi_escape
 
 
-TEST_GIT_REPO_NAME = 'hokusai-integration-test'
+NAME_OF_TEST_GIT_REPO = 'hokusai-integration-test'
 
 
 def exit_pytest_if_not_minikube(context):
@@ -41,18 +41,18 @@ def pytest_configure(config):
   # clone test git repo
   os.chdir('test/integration/fixtures')
   # skip cloning if already cloned and no force
-  if os.path.isdir(TEST_GIT_REPO_NAME) and os.environ.get('FORCE_CLONE') != '1':
+  if os.path.isdir(NAME_OF_TEST_GIT_REPO) and os.environ.get('FORCE_CLONE') != '1':
     return
-  shutil.rmtree(TEST_GIT_REPO_NAME, ignore_errors=True)
-  git.Git(".").clone(f"https://github.com/artsy/{TEST_GIT_REPO_NAME}.git")
+  shutil.rmtree(NAME_OF_TEST_GIT_REPO, ignore_errors=True)
+  git.Git(".").clone(f"https://github.com/artsy/{NAME_OF_TEST_GIT_REPO}.git")
 
 
 ## autouse fixtures
 
 @pytest.fixture(scope="session", autouse=True)
 def cd_into_test_git_repo():
-  os.chdir(TEST_GIT_REPO_NAME)
+  os.chdir(NAME_OF_TEST_GIT_REPO)
 
 @pytest.fixture(autouse=True)
-def test_git_repo_name():
-  return TEST_GIT_REPO_NAME
+def name_of_test_git_repo():
+  return NAME_OF_TEST_GIT_REPO

test/integration/test_base.py

@@ -13,8 +13,8 @@
 # working directory is set to repo dir, also in conftest.py
 
 def describe_git_repo_for_test():
-  def operating_in_test_git_repo_clone_dir(test_git_repo_name):
-    assert os.path.basename(os.getcwd()) == test_git_repo_name
+  def operating_in_test_git_repo_clone_dir(name_of_test_git_repo):
+    assert os.path.basename(os.getcwd()) == name_of_test_git_repo
   def operating_on_main_branch():
     repo = Repo(os.getcwd())
     heads = repo.heads

test/integration/test_staging.py

@@ -62,22 +62,21 @@ def it_reports_no_changes():
     assert 'Updated Kubernetes environment' in resp.stdout
 
 def describe_refresh():
-  def it_times_out():
-    # expect timeout due to minikube lacking ECR image pull permission
-    with pytest.raises(TimeoutExpired):
-      resp = subprocess.run(
-        'hokusai staging refresh',
-        capture_output=True,
-        shell=True,
-        text=True,
-        timeout=10
-      )
-      if resp.returncode != -9:
-        print(resp.stderr)
-      assert resp.returncode == -9
-      assert 'Refreshing hokusai-integration-test-web' in resp.stdout
-      assert 'Waiting for refresh to complete' in resp.stdout
-      assert 'Waiting for deployment "hokusai-integration-test-web" rollout to finish' in resp.stdout
+  def it_reports_success():
+    resp = subprocess.run(
+      'hokusai staging refresh',
+      capture_output=True,
+      shell=True,
+      text=True,
+      timeout=30
+    )
+    if resp.returncode != 0:
+      print(resp.stderr)
+    assert resp.returncode == 0
+    assert 'Refreshing hokusai-integration-test-web' in resp.stdout
+    assert 'Waiting for refresh to complete' in resp.stdout
+    assert 'Waiting for deployment "hokusai-integration-test-web" rollout to finish' in resp.stdout
+    assert 'deployment "hokusai-integration-test-web" successfully rolled out' in resp.stdout
 
 def describe_delete():
   def it_reports_deleted():