Merge requirements files and document dependency management (#21838)

mozilla · Feb 21, 2024 · 04cf44b · 04cf44b
1 parent e374a72
commit 04cf44b
Show file tree

Hide file tree

Showing 14 changed files with 487 additions and 778 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -450,15 +450,6 @@ commands:
       wait_services:
         type: boolean
         default: true
-      install_python_test_dependencies:
-        type: boolean
-        default: true
-      install_node_dependencies:
-        type: boolean
-        default: false
-      install_python_dev_dependencies:
-        type: boolean
-        default: false
     steps:
       - better_checkout
       - run:
@@ -497,32 +488,15 @@ commands:
             echo export NPM_CONFIG_PREFIX=/deps/ >> $BASH_ENV
             echo export CC=\"`python -c 'import sysconfig; print(sysconfig.get_config_var("CC"))'`\" >> $BASH_ENV
             cat $BASH_ENV
-      - when:
-          condition: << parameters.install_python_test_dependencies >>
-          steps:
-            - run:
-                name: Install Python dependencies
-                command: |
-                  make install_python_test_dependencies
-      - when:
-          condition: << parameters.install_python_dev_dependencies >>
-          steps:
-            - run:
-                name: Install Python dependencies
-                command: |
-                  make install_python_dev_dependencies
-      - when:
-          condition: << parameters.install_node_dependencies >>
-          steps:
-            - run:
-                name: Install Node dependencies
-                command: |
-                  sudo mkdir /deps
-                  sudo chown circleci /deps
-                  ACTUAL_CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"
-                  ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package.json /deps/package.json
-                  ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package-lock.json /deps/package-lock.json
-                  make install_node_dependencies
+      - run:
+            name: Install Python and Node dependencies
+            command: |
+              sudo mkdir /deps
+              sudo chown circleci /deps
+              ACTUAL_CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"
+              ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package.json /deps/package.json
+              ln -s ${ACTUAL_CIRCLE_WORKING_DIRECTORY}/package-lock.json /deps/package-lock.json
+              make update_deps
       # should be executed after all python install commands
       - run: pyenv rehash
 
@@ -537,8 +511,7 @@ jobs:
   amo-lib-locales-and-signing:
     <<: *defaults-with-autograph
     steps:
-      - setup_container:
-          install_node_dependencies: true
+      - setup_container
       - run:
           command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/amo/ src/olympia/lib/ src/olympia/signing
           environment:
@@ -555,10 +528,7 @@ jobs:
   assets:
     <<: *defaults-with-services
     steps:
-      - setup_container:
-          install_python_test_dependencies: false
-          install_python_dev_dependencies: true
-          install_node_dependencies: true
+      - setup_container
       - run: make update_assets
       - run:
           command: pytest -m "static_assets" -v src/olympia/
@@ -569,8 +539,6 @@ jobs:
     steps:
       - setup_container:
           wait_services: false
-          install_node_dependencies: true
-      - run: make install_python_codestyle_dependencies
       - run: pyenv rehash
       - run: make lint-codestyle
 
@@ -579,17 +547,13 @@ jobs:
     steps:
       - setup_container:
           wait_services: false
-          install_node_dependencies: true
-          install_python_dev_dependencies: true
       - run : make extract_locales
       - run: git diff
 
-
   devhub:
     <<: *defaults-with-services
     steps:
-      - setup_container:
-          install_node_dependencies: true
+      - setup_container
       - run:
           command: pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/devhub/
 
@@ -598,15 +562,12 @@ jobs:
     steps:
       - setup_container:
           wait_services: false
-          install_python_test_dependencies: false
-      - run: pip install --no-deps -r requirements/docs.txt
       - run: make docs
 
   main:
     <<: *defaults-with-services
     steps:
-      - setup_container:
-          install_node_dependencies: true
+      - setup_container
       - run:
           command: |
             pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \

diff --git a/Dockerfile b/Dockerfile
@@ -90,7 +90,7 @@ ENV PATH $PYTHONUSERBASE/bin:$PATH
 ENV NPM_CONFIG_PREFIX=/deps/
 RUN ln -s ${HOME}/package.json /deps/package.json \
     && ln -s ${HOME}/package-lock.json /deps/package-lock.json \
-    && make update_deps
+    && make update_deps_prod
 
 WORKDIR ${HOME}
 

diff --git a/Makefile-docker b/Makefile-docker
@@ -75,46 +75,27 @@ populate_data: ## populate a new database
 	# Now that addons have been generated, reindex.
 	$(PYTHON_COMMAND) manage.py reindex --force --noinput
 
-.PHONY: cleanup_python_build_dir
-cleanup_python_build_dir:
+.PHONY: update_deps_base
+update_deps_base: ## update the python and node dependencies
 	# Work arounds "Multiple .dist-info directories" issue.
 	rm -rf /deps/build/*
-
-.PHONY: install_python_olympia_module
-install_python_olympia_module:
 	# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
 	$(PIP_COMMAND) install --no-use-pep517 -e .
-
-.PHONY: install_python_codestyle_dependencies
-install_python_codestyle_dependencies:
-	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/codestyle.txt
-
-.PHONY: install_python_test_dependencies
-install_python_test_dependencies: install_python_olympia_module
-	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/system.txt
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
 	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/tests.txt
-
-.PHONY: install_python_dev_dependencies
-install_python_dev_dependencies: install_python_test_dependencies install_python_codestyle_dependencies
-	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/docs.txt
 
-.PHONY: install_node_dependencies
-install_node_dependencies: install_node_js copy_node_js
-
-.PHONY: install_node_js
-install_node_js:
 	npm install $(NPM_ARGS)
-
-.PHONY: copy_node_js
-copy_node_js:
 	for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done
 	for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done
 	for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done
 
 .PHONY: update_deps
-update_deps: cleanup_python_build_dir install_python_dev_dependencies install_node_dependencies ## update the python and node dependencies
+update_deps: update_deps_base ## update the python and node dependencies for development
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
+
+.PHONY: update_deps_prod
+update_deps_prod: update_deps_base ## update the python and node dependencies for production
+	npm prune --omit=dev
 
 .PHONY: update_db
 update_db: ## run the database migrations

diff --git a/docs/topics/development/dependencies.md b/docs/topics/development/dependencies.md
@@ -0,0 +1,63 @@
+# Project Dependencies
+
+This document describes how to add/upgrade dependencies in the project.
+We use pip to manage dependencies and hashin to lock versions. We use npm to manage frontend dependencies.
+
+## Python
+
+### Adding Python Dependencies
+
+We have 2 requirements files for python dependencies:
+
+- prod.txt
+- dev.txt
+
+Prod dependencies are used by our django app in runtime.
+They are strictly required to be installed in the production environment.
+
+Dev dependencies are used by our django app in development or by tools we use for linting, testing, etc.
+
+> If you add just the package name the script will automatically get the latest version for you.
+
+```bash
+hashin -r <requirements file> <dependency>
+```
+
+This will add hashes and sort the requirements for you adding comments to
+show any package dependencies.
+
+When it's run check the diff and make edits to fix any issues before
+submitting a PR with the additions.
+
+### Upgrading Python Dependencies
+
+We mostly rely on dependabot for this. TBD Add more details.
+
+## Frontend
+
+### Adding Frontend Dependencies
+
+We use npm to manage frontend dependencies. To add a new dependency, use the following command:
+
+```bash
+npm install [package]@[version] --save --save-dev
+```
+
+NPM is a fully featured package manager and so you can use the standard CLI.
+
+## Updating/Installing dependencies
+
+To update/install all dependencies, run the following command:
+
+```bash
+make update_deps
+```
+
+This will install all python and frontend dependencies. It also ensures olympia is installed locally.
+By default this command will run in a docker container, but you can run it on a host by targetting the Makefile-docker
+
+```bash
+make -f Makefile-docker update_deps
+```
+
+This is used in github actions for example that do not need a full container to run.
diff --git a/docs/topics/development/deps.rst b/docs/topics/development/deps.rst
diff --git a/docs/topics/development/index.rst b/docs/topics/development/index.rst
@@ -7,7 +7,7 @@ Development
 
    tests
    debugging
-   deps
+   dependencies
    error_pages
    testing
    style

diff --git a/requirements/codestyle.txt b/requirements/codestyle.txt