From 50b7248c9519ef2027ae0535ec37cb089c72ebbd Mon Sep 17 00:00:00 2001
From: Hannes Probst <hannes@deepnote.com>
Date: Thu, 21 Nov 2024 11:51:03 +0100
Subject: [PATCH 1/5] Use build-cache from main in PRs

---
 .circleci/config.yml | 61 ++++++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 25 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d485437..07a2eac 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -46,13 +46,16 @@ jobs:
       - run:
           name: Build and push base image
           command: |
+            REPOSITORY="deepnote/python"
+            TAG="${PYTHON_VERSION}-base"
+            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/base/Dockerfile.base \
-              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} \
-              --tag=docker.io/deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}} \
-              --cache-from type=registry,ref=docker.io/deepnote/python:base \
-              --cache-from type=registry,ref=docker.io/deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-buildcache,mode=max \
-              --cache-to type=registry,ref=docker.io/deepnote/python:base${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --tag=docker.io/deepnote/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
               --progress plain \
               --platform linux/amd64 \
               --provenance=false \
@@ -77,16 +80,18 @@ jobs:
           name: Build and push python image
           command: |
             REPOSITORY="deepnote/python"
-            TAG="${PYTHON_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            TAG="${PYTHON_VERSION}"
+            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/python/Dockerfile.python \
-              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG} \
-              --tag=docker.io/${REPOSITORY}:${TAG} \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST} \
               --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
               --build-arg PYTHON_VERSION_PATCH=${PYTHON_VERSION_PATCH} \
-              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max \
-              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
               --progress plain \
               --platform linux/amd64 \
               --provenance=false \
@@ -111,15 +116,17 @@ jobs:
           name: Build and push python-datascience image
           command: |
             REPOSITORY="deepnote/python"
-            TAG="${PYTHON_VERSION}-datascience${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            TAG="${PYTHON_VERSION}"
+            TAG_SUFFIX="-datascience${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/datascience/Dockerfile.datascience \
-              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG} \
-              --tag=docker.io/${REPOSITORY}:${TAG} \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST} \
               --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
-              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max \
-              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
               --progress plain \
               --platform linux/amd64 \
               --provenance=false \
@@ -144,15 +151,17 @@ jobs:
           name: Build and push python-conda image
           command: |
             REPOSITORY="deepnote/python"
-            TAG="${PYTHON_VERSION}-conda${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            TAG="${PYTHON_VERSION}"
+            TAG_SUFFIX="-conda${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/conda/Dockerfile.conda \
-              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG} \
-              --tag=docker.io/${REPOSITORY}:${TAG} \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --build-arg CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST} \
               --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
-              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max \
-              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
               --progress plain \
               --platform linux/amd64 \
               --provenance=false \
@@ -183,16 +192,18 @@ jobs:
           name: Build and push python image
           command: |
             REPOSITORY="deepnote/tensorflow"
-            TAG="${TF_VERSION}${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-gpu"
+            TAG="${TF_VERSION}"
+            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-gpu"
             docker buildx build \
               --file=./gpu/Dockerfile \
-              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG} \
-              --tag=docker.io/${REPOSITORY}:${TAG} \
+              --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --build-arg CUDA_IMAGE_TAG=${CUDA_TAG} \
               --build-arg TF_VERSION=${TF_VERSION} \
               --build-arg PYTHON_VER=${PY_VERSION} \
-              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max \
-              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
+              --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
+              --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \
               --progress plain \
               --platform linux/amd64 \
               --provenance=false \

From 09a9c0cd67ad18f7e5dcd2018a339a67d9be6741 Mon Sep 17 00:00:00 2001
From: Hannes Probst <hannes@deepnote.com>
Date: Thu, 21 Nov 2024 11:56:49 +0100
Subject: [PATCH 2/5] No python version needed for base

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 07a2eac..198b5fb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -47,7 +47,7 @@ jobs:
           name: Build and push base image
           command: |
             REPOSITORY="deepnote/python"
-            TAG="${PYTHON_VERSION}-base"
+            TAG="base"
             TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/base/Dockerfile.base \

From 85c1f075fd0ea590070a3c518d87efc0322272b3 Mon Sep 17 00:00:00 2001
From: Hannes Probst <hannes@deepnote.com>
Date: Thu, 21 Nov 2024 12:02:34 +0100
Subject: [PATCH 3/5] Fix gpu tag-suffix

---
 .circleci/config.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 198b5fb..b41dde4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -186,14 +186,13 @@ jobs:
     steps:
       - checkout
       - docker_login
-      - split_python_version # This provides PYTHON_VERSION
       - setup_buildkit_builder
       - run:
           name: Build and push python image
           command: |
             REPOSITORY="deepnote/tensorflow"
-            TAG="${TF_VERSION}"
-            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}-gpu"
+            TAG="${TF_VERSION}-gpu"
+            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./gpu/Dockerfile \
               --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \

From 67d6cb136dd3ee0450cc59032781e7da8a92f51b Mon Sep 17 00:00:00 2001
From: Hannes Probst <hannes@deepnote.com>
Date: Thu, 21 Nov 2024 12:15:30 +0100
Subject: [PATCH 4/5] Fix repository url

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b41dde4..aca83e5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -52,7 +52,7 @@ jobs:
             docker buildx build \
               --file=./python/base/Dockerfile.base \
               --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
-              --tag=docker.io/deepnote/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
+              --tag=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
               --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}-buildcache \
               --cache-from type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache \
               --cache-to type=registry,ref=docker.io/${REPOSITORY}:${TAG}${TAG_SUFFIX}-buildcache,mode=max,image-manifest=true,oci-mediatypes=true \

From 3785230cd1ebad01b9adc0e8e2e825743b9d9345 Mon Sep 17 00:00:00 2001
From: Hannes Probst <hannes@deepnote.com>
Date: Thu, 21 Nov 2024 12:27:48 +0100
Subject: [PATCH 5/5] Streamline tag usage in cache

---
 .circleci/config.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index aca83e5..771c12b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -116,8 +116,8 @@ jobs:
           name: Build and push python-datascience image
           command: |
             REPOSITORY="deepnote/python"
-            TAG="${PYTHON_VERSION}"
-            TAG_SUFFIX="-datascience${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            TAG="${PYTHON_VERSION}-datascience"
+            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/datascience/Dockerfile.datascience \
               --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \
@@ -151,8 +151,8 @@ jobs:
           name: Build and push python-conda image
           command: |
             REPOSITORY="deepnote/python"
-            TAG="${PYTHON_VERSION}"
-            TAG_SUFFIX="-conda${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
+            TAG="${PYTHON_VERSION}-conda"
+            TAG_SUFFIX="${CIRCLE_PULL_REQUEST:+-ra-${CIRCLE_PULL_REQUEST##*/}}"
             docker buildx build \
               --file=./python/conda/Dockerfile.conda \
               --tag=978928340082.dkr.ecr.us-east-1.amazonaws.com/${REPOSITORY}:${TAG}${TAG_SUFFIX} \