Improve docker entrypoints

- Separate develop from test dependencies
- Use entrypoints to run CI/CD tests
- Fix #88
- Optimize CI/CD pipeline by restoring cache in build deps

.circleci/config.yml

@@ -39,6 +39,7 @@ executors:
       POSTGRES_DB: postgres
       POSTGRES_PASSWORD: ""
       NIX_BUILD_SHELL: "bash"
+      ENTRYPOINT: "/app/docker-entrypoints/test.sh"
 
 jobs:
 
@@ -54,10 +55,21 @@ jobs:
 
       - checkout
 
+      - restore_cache:
+          keys:
+            - nix-deps-v1-{{ checksum "shell.nix" }}
+            - nix-deps-v1-
+
+      - restore_cache:
+          name: "Restoring Cache - Python"
+          keys:
+            - backend-deps-v3-{{ checksum "poetry.lock" }}
+            - backend-deps-v3-
+
       - run:
           name: Build nix shell
           command: |
-            nix-shell --pure --run "python --version"
+            $ENTRYPOINT python --version
 
       - save_cache:
           key: nix-deps-v1-{{ checksum "shell.nix" }}
@@ -76,7 +88,7 @@ jobs:
       - run:
           name: Install dependencies
           command: |
-            nix-shell --pure --run "poetry install"
+            $ENTRYPOINT poetry install
 
       - save_cache:
           key: backend-deps-v3-{{ checksum "poetry.lock" }}
@@ -109,14 +121,14 @@ jobs:
       - run:
           name: Run Linters
           command: |
-            nix-shell --pure --run "poetry run pre-commit install -f --hook-type pre-commit"
-            nix-shell --pure --run "poetry run pre-commit install -f --hook-type pre-push"
-            nix-shell --pure --run "make lint all=true"
+            $ENTRYPOINT pre-commit install -f --hook-type pre-commit
+            $ENTRYPOINT pre-commit install -f --hook-type pre-push
+            $ENTRYPOINT make lint all=true
 
       - run:
           name: run mypy
           command: |
-            nix-shell --pure --run "make types"
+            $ENTRYPOINT make types
 
 
   Backend Test:
@@ -147,7 +159,7 @@ jobs:
           command: |
             for i in `seq 1 10`;
             do
-              nix-shell --pure --run "nc -z localhost 5432" && echo Success && exit 0
+              $ENTRYPOINT nc -z localhost 5432 && echo Success && exit
               echo -n .
               sleep 1
             done
@@ -157,21 +169,22 @@ jobs:
           name: Create testing db
           # TODO: add --pure? but then PGHOST is missing
           command: |
-            nix-shell --run ".docker/devdb.sh && .docker/testdb.sh"
+            $ENTRYPOINT .docker/devdb.sh
+            $ENTRYPOINT .docker/testdb.sh
 
       - run:
           name: Test development.ini
           command: |
-            nix-shell --run ".docker/devdb.sh"
-            nix-shell --pure --run "poetry run pshell etc/development.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh"
+            $ENTRYPOINT .docker/devdb.sh
+            $ENTRYPOINT pshell etc/development.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh
 
       - run:
           name: Checking consistency between alembic revision and conduit models
           command: |
-            nix-shell --pure --run "make devdb"
+            $ENTRYPOINT make devdb
             # As of now, the word count in an empty migration script is: 84
             EXPECTED_WORD_COUNT=84
-            PRESENT_WORD_COUNT=$(cat $(nix-shell --pure --run "poetry run alembic -c etc/alembic.ini -x ini=etc/development.ini revision --autogenerate -m 'Test Consistency'" | cut -d' ' -f2) | wc -w)
+            $ENTRYPOINT alembic -c etc/alembic.ini -x ini=etc/development.ini revision --autogenerate -m 'Test Consistency'" | cut -d' ' -f2) | wc -w
             if [ $EXPECTED_WORD_COUNT != $PRESENT_WORD_COUNT ]; then
               echo "You probably forgot to generate and commit DB migration...!"
               false
@@ -180,12 +193,12 @@ jobs:
       - run:
           name: Test production.ini
           command: |
-            nix-shell --pure --run "JWT_SECRET=secret DATABASE_URL=postgresql://conduit_dev@localhost/conduit_dev poetry run pshell etc/production.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh"
+            /app/docker-entrypoints/run.sh env JWT_SECRET=secret DATABASE_URL=postgresql://conduit_dev@localhost/conduit_dev pshell etc/production.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh
 
       - run:
           name: Run unit tests
           command: |
-            nix-shell --pure --run "make unit"
+            $ENTRYPOINT make unit
 
 
   ######################################
@@ -219,7 +232,7 @@ jobs:
           command: |
             for i in `seq 1 10`;
             do
-              nix-shell --pure --run "nc -z localhost 5432" && echo Success && exit 0
+              $ENTRYPOINT nc -z localhost 5432 && echo Success && exit
               echo -n .
               sleep 1
             done
@@ -229,7 +242,7 @@ jobs:
           name: Run the tests
           # TODO: add --pure? but then PGHOST is missing
           command: |
-            nix-shell --run ".docker/devdb.sh"
-            nix-shell --run "make devdb"
-            nix-shell --run "poetry run pserve etc/development.ini &"
-            nix-shell --run "make postman-tests"
+            $ENTRYPOINT .docker/devdb.sh
+            $ENTRYPOINT make devdb
+            $ENTRYPOINT pserve etc/development.ini &
+            $ENTRYPOINT make postman-tests

Dockerfile

@@ -6,36 +6,39 @@ WORKDIR /app
 
 # Only copy files relevant for the nix-shell in an earlier layer to
 # have this be cached when those files don't change
-COPY --chown=builder ./shell.nix ./nixpkgs.json ./pyproject.toml ./poetry.lock ./poetry.toml ./
-RUN nix-shell --show-trace --argstr type build
+COPY --chown=builder ./shell.nix ./nixpkgs.json ./
+RUN nix-shell --argstr type build
 
-COPY --chown=builder . /app
+COPY --chown=builder . ./
 RUN \
-	# Clean all untracked git files in case this is built in an unclean tree
-	nix-shell --argstr type build --run "git clean -ffxd || true" \
-	&& nix-shell --argstr type build --run "su builder -c 'poetry install --no-dev'" \
-	&& nix-shell --argstr type run --run "nix-env -iA nixpkgs.bashInteractive"
+  # Clean all untracked git files in case this is built in an unclean tree
+  nix-shell --argstr type build --run "su builder -c 'if [ -d .git ]; then git clean -ffxd; fi'" \
+  # Install Python dependencies
+  && nix-shell --argstr type build --run "su builder -c 'poetry install -n --no-dev'" \
+  # Clean up garbage in a nix-shell with runtime dependencies, meaning everything
+  # not needed in it should be removed
+  && /app/docker-entrypoints/run.sh nix-env -iA nixpkgs.bashInteractive \
+  && /app/docker-entrypoints/run.sh nix-collect-garbage -d
 
 # Multi-stage build to only propagate what we need
 FROM nixos/nix
 
 # Heroku stack images have these set, so we need to set them too
 # to make Gunicorn work as expected
 ENV \
-	FORWARDED_ALLOW_IPS="*" \
-	GUNICORN_CMD_ARGS="--access-logfile -"
+    FORWARDED_ALLOW_IPS="*" \
+    GUNICORN_CMD_ARGS="--access-logfile -" \
+    NIX_BUILD_SHELL=/nix/var/nix/profiles/default/bin/bash
 
 COPY --from=build /nix /nix
 # Needed such that the nixpkgs fetched with fetchTarball isn't redownloaded
 COPY --from=build /root/.cache/nix/tarballs /root/.cache/nix/tarballs
-COPY --from=build --chown=root /app /app
+COPY --from=build /app /app
+
 WORKDIR /app
 
-# Create an entrypoint
-RUN \
-	echo $'#!/usr/bin/env nix-shell\n#!nix-shell --argstr type run -i bash /app/shell.nix\npoetry run "$@"' \
-	| tee /usr/local/bin/docker-entrypoint.sh \
-	&& chmod +x /usr/local/bin/docker-entrypoint.sh
+# Set the entrypoint
+ENTRYPOINT ["/app/docker-entrypoints/run.sh"]
 
-ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+# Default command
 CMD ["bash"]

docker-entrypoints/README.md

@@ -0,0 +1,18 @@
+Docker Entrypoints
+==================
+
+Entrypoints are used and a wrapper around the commands we want to run.
+Here we use entrypoint to invoke a shell with the appropriate environment
+and dependencies available to run the given commands.
+
+e.g.
+
+```bash
+/app/docker-entrypoints/run.sh some command
+```
+
+We set the entrypoint in a docker image using the `ENTRYPOINT` instruction.
+
+We currently have three entrypoints: `run.sh`, `test.sh` and `develop.sh`.
+
+For more info, read the comments in the respective files.

docker-entrypoints/develop.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --argstr type develop -i bash /app/shell.nix
+
+# This is the default environment for nix-shell and is used for
+# developing/debugging the app. It contains all the dependencies from `test`
+# and some extra dependencies such as `tmux`, `vim` etc.
+
+# This entrypoint can also be used in Heroku shell or CircleCI for debugging.
+# e.g. /app/docker-entrypoints/develop.sh vim
+
+poetry run "$@"

docker-entrypoints/run.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --argstr type run -i bash /app/shell.nix
+
+# This entrypoint is generally used by Heroku to run the already built app.
+# The given commands are run in an environment that has the least set of
+# dependencies available - only binaries and libraries required to run the
+# service. e.g. poetry, psql etc.
+#
+# This entrypoint also creates the SSH keys from the run-time variables in Heroku.
+
+poetry run "$@"

docker-entrypoints/test.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env nix-shell
+#!nix-shell --argstr type test -i bash /app/shell.nix
+
+# This entrypoint is generally used by the CI/CD pipeline.
+# This runs the given commands in an environment that has all the dependencies
+# from `run` and some additional dependencies to run the tests.
+# e.g. geckodriver for browser tests.
+
+poetry run "$@"

shell.nix

@@ -20,7 +20,8 @@ let
   }) { config = {}; overlays = []; };
 
   dependencies = let mapping = {
-    develop = developDeps ++ buildDeps ++ runDeps;
+    develop = developDeps ++ testDeps ++ buildDeps ++ runDeps;
+    test = testDeps ++ buildDeps ++ runDeps;
     build = buildDeps ++ runDeps;
     run = runDeps;
   }; in mapping.${type} or (throw
@@ -29,6 +30,21 @@ let
   stdenv = if type == "develop" then pkgs.stdenv else pkgs.stdenvNoCC;
 
   developDeps = with pkgs; [
+    heroku
+    vim
+    micro
+    tmux
+    curl
+  ]
+
+  # The watchdog Python lib has a few extra requirements on Darwin (MacOS)
+  # Taken from https://github.com/NixOS/nixpkgs/blob/d72887e0d28a98cc6435bde1962e2b414224e717/pkgs/development/python-modules/watchdog/default.nix#L20
+  ++ lib.optionals pkgs.stdenv.isDarwin [
+    pkgs.darwin.apple_sdk.frameworks.CoreServices
+  ];
+
+  # These are needed to run tests in CI/CD
+  testDeps = with pkgs; [
     gnumake
     nodejs
     b2sum
@@ -42,10 +58,12 @@ let
     jq
   ]
 
-  # The watchdog Python lib has a few extra requirements on Darwin (MacOS)
-  # Taken from https://github.com/NixOS/nixpkgs/blob/d72887e0d28a98cc6435bde1962e2b414224e717/pkgs/development/python-modules/watchdog/default.nix#L20
-  ++ lib.optionals pkgs.stdenv.isDarwin [
-    pkgs.darwin.apple_sdk.frameworks.CoreServices
+  # Currently, both firefox and firefox-bin are broken on Darwin (MacOS)
+  # so if you are on a MacBook, you have to manually install firefox.
+  # If https://github.com/NixOS/nixpkgs/issues/53979 gets fixed,
+  # we can remove this if.
+  ++ lib.optionals (!pkgs.stdenv.isDarwin) [
+    pkgs.firefox
   ];
 
   # These are needed to build the production artifacts
@@ -56,7 +74,6 @@ let
   # Only these dependencies are needed to run in production
   runDeps = with pkgs; [
     ( unstable.poetry.override { python = unstable.python38; } )
-    curl                      # For downloading other tools via Heroku shell
     unstable.postgresql_12    # For interacting with the database
   ];
 in