airbyte-ci: add /var/lib/docker cachemount

airbyte-ci/connectors/pipelines/README.md

@@ -408,7 +408,8 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
-| 2.5.3   | [#32090](https://github.com/airbytehq/airbyte/pull/32090)  | Do not cache `docker login`.                            |
+| 2.5.5   | [#TBD](https://github.com/airbytehq/airbyte/pull/TBD)  | Create cache mount for `/var/lib/docker` to store images in `dind` context.                            |
+| 2.5.4   | [#32090](https://github.com/airbytehq/airbyte/pull/32090)  | Do not cache `docker login`.                            |
 | 2.5.3   | [#31974](https://github.com/airbytehq/airbyte/pull/31974)  | Fix latest CDK install and pip cache mount on connector install.                            |
 | 2.5.2   | [#31871](https://github.com/airbytehq/airbyte/pull/31871)  | Deactivate PR comments, add HTML report links to the PR status when its ready.                            |
 | 2.5.1   | [#31774](https://github.com/airbytehq/airbyte/pull/31774)  | Add a docker configuration check on `airbyte-ci` startup.                                                 |

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -45,6 +45,7 @@
 DOCKER_HOST_NAME = "global-docker-host"
 DOCKER_HOST_PORT = 2375
 DOCKER_TMP_VOLUME_NAME = "shared-tmp"
+DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache"
 REPO = git.Repo(search_parent_directories=True)
 REPO_PATH = REPO.working_tree_dir
 STATIC_REPORT_PREFIX = "airbyte-ci"

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py

@@ -9,7 +9,7 @@
 from dagger import Client, Container, File, Secret
 from pipelines import consts
 from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext
-from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_TMP_VOLUME_NAME
+from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_TMP_VOLUME_NAME, DOCKER_VAR_LIB_VOLUME_NAME
 from pipelines.helpers.utils import sh_dash_c
 
 
@@ -26,7 +26,8 @@ def with_global_dockerd_service(
         Container: The container running dockerd as a service
     """
     dockerd_container = (
-        dagger_client.container().from_(consts.DOCKER_DIND_IMAGE)
+        dagger_client.container()
+        .from_(consts.DOCKER_DIND_IMAGE)
         # We set this env var because we need to use a non-default zombie reaper setting.
         # The reason for this is that by default it will want to set its parent process ID to 1 when reaping.
         # This won't be possible because of container-ception: dind is running inside the dagger engine.
@@ -49,6 +50,7 @@ def with_global_dockerd_service(
         # Expose the docker host port.
         .with_exposed_port(DOCKER_HOST_PORT)
         # Mount the docker cache volumes.
+        .with_mounted_cache("/var/lib/docker", dagger_client.cache_volume(DOCKER_VAR_LIB_VOLUME_NAME))
         .with_mounted_cache("/tmp", dagger_client.cache_volume(DOCKER_TMP_VOLUME_NAME))
     )
     if docker_hub_username_secret and docker_hub_password_secret:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "2.5.4"
+version = "2.5.5"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]