move build pipeline to circleci (#4923)

mozilla · Dec 7, 2021 · 2871ee4 · 2871ee4
1 parent c3ba0e8
commit 2871ee4
Show file tree

Hide file tree

Showing 39 changed files with 258 additions and 655 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,31 +1,73 @@
-version: 2
+version: 2.1
+orbs:
+  aws-cli: circleci/aws-cli@2.0
 jobs:
-  build:
+  lint:
     docker:
-      - image: circleci/python:3.9-buster
+      - image: circleci/python:3.9-bullseye
     steps:
       - checkout
       - run:
           name: Lint with pre-commit
           command: |
             sudo pip install --upgrade pre-commit==2.15.0
             pre-commit run --all-files
+  test-and-build:
+    docker:
+      - image: circleci/python:3.9-bullseye
+    environment:
+      AWS_DEFAULT_REGION: us-west-2
+    steps:
+      - checkout
       - setup_remote_docker:
-          version: 17.10.0-ce
+          version: 20.10.7
           docker_layer_caching: true
       - run:
-          name: Build docker images
-          command: make build-ci
-      - run:
-          name: Run mocha tests
-          command: make test-js-ci
+          name: Build test image
+          command: |
+            cp .env-build .env
+            ./bin/dc_ci.sh build --progress=plain test
       - run:
           # copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
           # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
           name: Copy synonym files to elasticsearch7
           command: |
-            ./bin/dc.sh up -d elasticsearch7
+            ./bin/dc_ci.sh up -d elasticsearch7
             docker cp ./kitsune/search/dictionaries/synonyms/. project_elasticsearch7_1:/usr/share/elasticsearch/config/synonyms
       - run:
           name: Run unit tests
-          command: make test-ci
+          command: ./bin/dc_ci.sh run test ./bin/run-unit-tests.sh
+      - run:
+          name: Run js tests
+          command: ./bin/dc_ci.sh run test ./bin/run-mocha-tests.sh
+      - when:
+          condition:
+            or:
+              - equal: [ main, << pipeline.git.branch >> ]
+              - equal: [ production, << pipeline.git.branch >> ]
+              - matches: { pattern: "^prod-.+$", value: << pipeline.git.branch >> }
+          steps:
+            - run:
+                name: Build prod image
+                command: ./bin/dc_ci.sh build --progress=plain prod
+            - run:
+                name: Push prod image
+                command: |
+                  echo "$DOCKER_PASSWORD" | docker login -u $DOCKER_USERNAME --password-stdin
+                  source docker/bin/set_git_env_vars.sh
+                  docker image tag mozilla/kitsune:prod-${GIT_COMMIT_SHORT} mozilla/kitsune:prod-latest
+                  docker image push mozilla/kitsune:prod-${GIT_COMMIT_SHORT}
+                  docker image push mozilla/kitsune:prod-latest
+                  docker logout
+            - aws-cli/setup
+            - run:
+                name: Upload staticfiles
+                command: |
+                  source docker/bin/set_git_env_vars.sh
+                  ./docker/bin/upload-staticfiles.sh
+workflows:
+  version: 2
+  lint-test-build:
+    jobs:
+      - lint
+      - test-and-build
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
 #-------------------------------------------------------------------------------------------------------------
 
-version: '3.4'
+version: '3.8'
 services:
   # Update this to the name of the service you want to work with in your docker-compose.yml file
   web:

diff --git a/.dockerignore b/.dockerignore
@@ -16,3 +16,5 @@ __pycache__
 .npm
 .vscode-server
 .cache
+.gnupg*
+styleguide
diff --git a/.env-build b/.env-build
@@ -7,6 +7,5 @@ CELERY_TASK_ALWAYS_EAGER=True
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=sqlite://
 DATABASE_READ_ONLY_URL=sqlite://
-ES7_URLS=elasticsearch:9200
 SESSION_COOKIE_SECURE=False
 SECRET_KEY=secret
diff --git a/.env-test b/.env-test
@@ -1,3 +1,4 @@
+TEST=True
 DEBUG=False
 # Some cron jobs are skipped on stage.
 STAGE=False
@@ -16,7 +17,6 @@ CACHE_URL=redis://redis:6379/3
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=mysql://root:kitsune@mariadb:3306/kitsune
 DB_CONN_MAX_AGE=0
-ES_URLS=elasticsearch:9200
 SECRET_KEY=secret
 REUSE_STATIC=1
 REUSE_DB=0

diff --git a/Dockerfile b/Dockerfile
@@ -1,17 +1,7 @@
-################################
-# Frontend dependencies builder
-#
-FROM node:12 AS frontend-base
-
-WORKDIR /app
-COPY ["./package.json", "./package-lock.json", "prepare_django_assets.js", "/app/"]
-COPY ./kitsune/sumo/static/sumo /app/kitsune/sumo/static/sumo
-RUN npm run production
-
-################################
-# Python dependencies builder
-#
-FROM python:3.9-buster AS base
+#######################
+# Common dependencies #
+#######################
+FROM python:3.9-bullseye AS base
 
 WORKDIR /app
 EXPOSE 8000
@@ -26,83 +16,84 @@ RUN python -m venv /venv
 RUN pip install --upgrade "pip==21.3.1"
 RUN useradd -d /app -M --uid 1000 --shell /usr/sbin/nologin kitsune
 
-RUN apt-get update && \
+RUN apt-get update && apt-get install apt-transport-https && \
+    curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
+    apt-get update && \
     apt-get install -y --no-install-recommends \
     gettext build-essential \
     libxml2-dev libxslt1-dev zlib1g-dev git \
     libjpeg-dev libffi-dev libssl-dev libxslt1.1 \
-    libmariadb3 mariadb-client && \
+    libmariadb3 mariadb-client \
+    optipng nodejs zip && \
     rm -rf /var/lib/apt/lists/*
 
 COPY ./requirements/*.txt /app/requirements/
+RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt
 
-RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt && \
-    pip install --no-cache-dir --require-hashes -r requirements/dev.txt
 
-ARG GIT_SHA=head
-ENV GIT_SHA=${GIT_SHA}
+#####################
+# Development image #
+#####################
+FROM base AS dev
 
+RUN pip install --no-cache-dir --require-hashes -r requirements/dev.txt
 
-################################
-# Developer image
-#
-FROM base AS base-dev
-RUN apt-get update && apt-get install apt-transport-https && \
-    curl -sL https://deb.nodesource.com/setup_12.x | bash -
-RUN apt-get update && apt-get install -y --no-install-recommends optipng nodejs zip && \
-    rm -rf /var/lib/apt/lists/*
 
+#########################
+# Frontend dependencies #
+#########################
+FROM base AS base-frontend
 
-################################
-# Fetch locales
-#
-FROM python:3.9-buster AS locales
+COPY package*.json .
+COPY prepare_django_assets.js .
+RUN npm run install-prod && npm run copy:protocol && npm run postinstall
 
-WORKDIR /app
+COPY kitsune/sumo/static/sumo/scss kitsune/sumo/static/sumo/scss
+RUN npm run build:scss && npm run build:postcss
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends  gettext
+COPY . .
+RUN cp .env-build .env && \
+    ./manage.py nunjucks_precompile
 
-ENV PATH="/venv/bin:$PATH"
 
-COPY --from=base /venv /venv
+########################
+# Testing dependencies #
+########################
+FROM base AS test-deps
 
-COPY . .
+RUN pip install --no-cache-dir --require-hashes -r requirements/test.txt
 
-ARG LOCALE_ENV=main
-ENV LOCALE_ENV=${LOCALE_ENV}
-RUN ./docker/bin/fetch-l10n-files.sh
-RUN ./scripts/compile-linted-mo.sh && \
-    find ./locale ! -name '*.mo' -type f -delete
 
-ARG GIT_SHA=head
-ENV GIT_SHA ${GIT_SHA}
+#################
+# Testing image #
+#################
+FROM base-frontend AS test
 
+COPY --from=test-deps /venv /venv
 
-################################
-# Staticfiles builder
-#
-FROM base-dev AS staticfiles
+RUN cp .env-test .env && \
+    ./manage.py compilejsi18n && \
+    ./manage.py collectstatic --noinput
 
-COPY --from=frontend-base --chown=kitsune:kitsune /app/assets /app/assets
-COPY --from=frontend-base --chown=kitsune:kitsune /app/node_modules /app/node_modules
-COPY --from=locales /app/locale /app/locale
 
-COPY . .
+##########################
+# Production dependences #
+##########################
+FROM base-frontend AS prod-deps
 
-RUN cp .env-build .env && \
-    ./manage.py nunjucks_precompile && \
+RUN ./scripts/l10n-fetch-lint-compile.sh && \
+    find ./locale ! -name '*.mo' -type f -delete && \
     ./manage.py compilejsi18n && \
     # minify jsi18n files:
     find jsi18n/ -name "*.js" -exec sh -c 'npx uglifyjs "$1" -o "${1%.js}-min.js"' sh {} \; && \
     ./manage.py collectstatic --noinput && \
     npx svgo -r -f static
 
 
-################################
-# Full prod image sans locales
-#
-FROM python:3.9-slim-buster AS full-no-locales
+##########################
+# Clean production image #
+##########################
+FROM python:3.9-slim-bullseye AS prod
 
 WORKDIR /app
 
@@ -113,32 +104,25 @@ ENV LANG=C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 
+RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
+
+COPY --from=prod-deps --chown=kitsune:kitsune /venv /venv
+COPY --from=prod-deps --chown=kitsune:kitsune /app/locale /app/locale
+COPY --from=prod-deps --chown=kitsune:kitsune /app/static /app/static
+
+COPY --chown=kitsune:kitsune . .
+
+# apt-get after copying everything to ensure we're always getting the latest packages in the prod image
 RUN apt-get update && \
+    apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
     libmariadb3 optipng mariadb-client \
     libxslt1.1 && \
     rm -rf /var/lib/apt/lists/*
 
-RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
-
-COPY --from=base --chown=kitsune:kitsune /venv /venv
-COPY --from=staticfiles --chown=kitsune:kitsune /app/static /app/static
-
-COPY --chown=kitsune:kitsune . .
-
 RUN mkdir /app/media && chown kitsune:kitsune /app/media
 
 USER kitsune
 
 ARG GIT_SHA=head
 ENV GIT_SHA ${GIT_SHA}
-
-
-################################
-# Full final prod image
-#
-FROM full-no-locales AS full
-
-USER root
-COPY --from=locales --chown=kitsune:kitsune /app/locale /app/locale
-USER kitsune