Implement All CircleCI tests in github actions (#22250)

* TMP: move test definition to make file TMP: squash * TMP: add the github version * TMP: squash (no need to parallelize this test)
mozilla · May 31, 2024 · ef033d0 · ef033d0
1 parent 27c102f
commit ef033d0
Show file tree

Hide file tree

Showing 4 changed files with 284 additions and 29 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -546,14 +546,14 @@ jobs:
     steps:
       - setup_container
       - run:
-          command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/addons/ src/olympia/versions/ src/olympia/files/ src/olympia/ratings/
+          command: make test_addons_versions_files_ratings
 
   amo-lib-locales-and-signing:
     <<: *defaults-with-autograph
     steps:
       - setup_container
       - run:
-          command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/amo/ src/olympia/lib/ src/olympia/signing
+          command: make test_amo_lib_locales_and_signing
           environment:
             AUTOGRAPH_SERVER_URL: http://127.0.0.1:5500
       # After having removed `tox`, we noticed that this job was a lot slower.
@@ -563,16 +563,15 @@ jobs:
       - run: echo "export PATH=\"$(pyenv prefix)/bin:$PATH\"" >> $BASH_ENV
       - run: make compile_locales
       - run:
-          command: pytest -n 2 -m 'needs_locales_compilation' -v src/olympia/
+          command: make test_needs_locales_compilation
 
   assets:
     <<: *defaults-with-services
     steps:
       - setup_container
       - run: make update_assets
       - run:
-          command: pytest -m "static_assets" -v src/olympia/
-      - run: make run_js_tests
+          command: make test_static_assets
 
   codestyle:
     <<: *defaults
@@ -595,44 +594,26 @@ jobs:
     steps:
       - setup_container
       - run:
-          command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/devhub/
+          command: make test_devhub
 
   main:
     <<: *defaults-with-services
     steps:
       - setup_container
       - run:
-          command: |
-            pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \
-              -v src/olympia/ \
-              --ignore src/olympia/addons/ \
-              --ignore src/olympia/devhub/ \
-              --ignore src/olympia/files/ \
-              --ignore src/olympia/reviewers/ \
-              --ignore src/olympia/ratings/ \
-              --ignore src/olympia/amo/ \
-              --ignore src/olympia/lib/ \
-              --ignore src/olympia/signing \
-              --ignore src/olympia/versions/ \
-              --ignore src/olympia/zadmin
+          command: make test_main
 
   reviewers-and-zadmin:
     <<: *defaults-with-autograph
     steps:
       - setup_container
       - run:
-          command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/reviewers/ src/olympia/zadmin/
+          command: make test_reviewers_and_zadmin
           environment:
             AUTOGRAPH_SERVER_URL: http://127.0.0.1:5500
       - run:
-          name: internal_routes_allowed
-          command: |
-            # We need to change the setting in the file because we can't
-            # override an env variable here, and the next command requires
-            # `INTERNAL_ROUTES_ALLOWED` to be set to `True`.
-            sed -i 's/^INTERNAL_ROUTES_ALLOWED.*/INTERNAL_ROUTES_ALLOWED=True/' settings_test.py
-            pytest -m 'internal_routes_allowed' -v src/olympia/
-
+          name: test_internal_routes_allowed
+          command: make test_internal_routes_allowed
   es-tests:
     <<: *defaults-with-elasticsearch
     steps:
@@ -641,7 +622,7 @@ jobs:
           name: Wait for elasticsearch
           command: dockerize -wait tcp://localhost:9200 -timeout 1m
       - run:
-          command: pytest -m "es_tests and not needs_locales_compilation and not static_assets" -v src/olympia/
+          command: make test_es_tests
 
   # Add to a workflow, if you want to test the docker build in circleci
   build-image:

diff --git a/.github/workflows/verify-docker-image.yml b/.github/workflows/verify-docker-image.yml
@@ -70,6 +70,13 @@ jobs:
           run: |
             make check
 
+      - name: Codestyle
+        uses: ./.github/actions/run-docker
+        with:
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make lint-codestyle
+
   build_docs:
     runs-on: ubuntu-latest
 
@@ -139,3 +146,197 @@ jobs:
           fi
 
           make push_locales ARGS="--dry-run"
+
+  test_test_addons_versions_files_ratings:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_addons_versions_files_ratings)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_addons_versions_files_ratings
+
+  test_amo_lib_locales_and_signing:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+
+      - name: Test (test_amo_lib_locales_and_signing)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_amo_lib_locales_and_signing
+
+  test_needs_locales_compilation:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_needs_locales_compilation)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_needs_locales_compilation
+
+  test_static_assets:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_static_assets)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          # TODO: we should remove this once we
+          # a) update the asset tests to look in the static-assets folder
+          # b) copy the static file into the container also.
+          run: |
+            make update_assets
+            make test_static_assets
+
+  test_devhub:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_devhub)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_devhub
+
+  test_main:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_main)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_main
+
+  test_reviewers_and_zadmin:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_reviewers_and_zadmin)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_reviewers_and_zadmin
+
+  test_internal_routes_allowed:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_internal_routes_allowed)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_internal_routes_allowed
+
+  test_es_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Cache Dependencies Directory
+        uses: ./.github/actions/cache-deps
+
+      - name: Test (test_es_tests)
+        uses: ./.github/actions/run-docker
+        with:
+          services: ''
+          version: ${{ steps.build.outputs.version }}
+          run: |
+            make test_es_tests
diff --git a/Makefile-docker b/Makefile-docker
@@ -241,6 +241,78 @@ reload: ## force django code reload
 
 reload-uwsgi: reload
 
+.PHONY: test_addons_versions_files_ratings
+test_addons_versions_files_ratings:
+	pytest \
+		-n 2	\
+		-m 'not es_tests and not needs_locales_compilation and not static_assets' \
+		-v src/olympia/addons/ src/olympia/versions/ src/olympia/files/ src/olympia/ratings/
+
+.PHONY: test_amo_lib_locales_and_signing
+test_amo_lib_locales_and_signing:
+	pytest \
+		-n 2 \
+		-m 'not es_tests and not needs_locales_compilation and not static_assets' \
+		-v src/olympia/amo/ src/olympia/lib/ src/olympia/signing
+
+.PHONY: test_needs_locales_compilation
+test_needs_locales_compilation:
+	pytest \
+		-m 'needs_locales_compilation' \
+		-v src/olympia/
+
+.PHONY: test_static_assets
+test_static_assets: run_js_tests
+	pytest \
+		-m 'static_assets' \
+		-v src/olympia/
+
+.PHONY: test_devhub
+test_devhub:
+	pytest \
+		-n 2 \
+		-m 'not es_tests and not needs_locales_compilation and not static_assets' \
+		-v src/olympia/devhub/
+
+.PHONY: test_main
+test_main:
+	pytest \
+	-n 2 \
+	-m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \
+	-v src/olympia/ \
+	--ignore src/olympia/addons/ \
+	--ignore src/olympia/devhub/ \
+	--ignore src/olympia/files/ \
+	--ignore src/olympia/reviewers/ \
+	--ignore src/olympia/ratings/ \
+	--ignore src/olympia/amo/ \
+	--ignore src/olympia/lib/ \
+	--ignore src/olympia/signing \
+	--ignore src/olympia/versions/ \
+	--ignore src/olympia/zadmin
+
+.PHONY: test_reviewers_and_zadmin
+test_reviewers_and_zadmin:
+	pytest \
+		-n 2 \
+		-m 'not es_tests and not needs_locales_compilation and not static_assets' \
+		-v src/olympia/reviewers/ src/olympia/zadmin/
+
+.PHONY: test_internal_routes_allowed
+test_internal_routes_allowed:
+# We need to change the setting in the file because we can't
+# override an env variable here, and the next command requires
+# `INTERNAL_ROUTES_ALLOWED` to be set to `True`.
+	sed -i 's/^INTERNAL_ROUTES_ALLOWED.*/INTERNAL_ROUTES_ALLOWED=True/' settings_test.py
+	pytest -m 'internal_routes_allowed' -v src/olympia/
+
+.PHONY: test_es_tests
+test_es_tests:
+	pytest \
+		-m "es_tests and not needs_locales_compilation and not static_assets" \
+		-v src/olympia/
+
+
 .PHONY: test
 test: ## run the entire test suite
 	pytest $(APP) $(ARGS)

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,6 +4,7 @@ exclude = [
     "static",
     ".git",
     "*/migrations/*.py",
+    "build.py",
 ]
 line-length = 88