Add make targets for common tasks on dev env and support for env snap…

…shots (#667) * Add make targets for common tasks on dev env and support for env snapshots This PR is addresses AAH-387 adding more targets to Makefile Also some tweaks to Docker files and setup.py to enable running specific deps branches. Adds ability to set a global `suffix` so it is possible to have multiple environments already built and saved (snapshots) Adds utilities to run 2 instances of AAH in the same host. Issue: AAH-387 * Add django-extensions and more make targets Issue: AAH-387 * Add default values to INFO debug message No-Issue
ansible · Mar 12, 2021 · 1a541c6 · 1a541c6
1 parent a72c3df
commit 1a541c6
Show file tree

Hide file tree

Showing 10 changed files with 155 additions and 18 deletions.
diff --git a/.compose.env.example b/.compose.env.example
@@ -7,9 +7,29 @@ COMPOSE_PROFILE=standalone
 # DEV_SOURCE_PATH='pulpcore:pulp_ansible:galaxy_ng'
 DEV_SOURCE_PATH='galaxy_ng'
 
+## Determines if pip should install packages from requirements/requirements.<COMPOSE_PROFILE>.txt
 # set to `0` to bypass requirements and install only from setup.py for each DEV_SOURCE_PATH
+# and unpin galaxy_ng/setup.py requirements
 LOCK_REQUIREMENTS=1
 
 # If you want to run the UI, clone https://github.com/ansible/ansible-hub-ui
 # and set this to the path for the UI
 # ANSIBLE_HUB_UI_PATH='/absolute/path/to/ansible-hub-ui'
+
+# If you want to keep your current running environment but you need
+# to build and run a separate env, this variable can set image and volume
+# suffix to allow snapshoting of the environments.
+#
+# Tip: This can also be passed directly as a compose env argument
+#      but in that case every ./compose call must have it
+#      e.g:
+#        ./compose build -e DEV_IMAGE_SUFFIX=_working_on_pr_123
+#        DEV_IMAGE_SUFFIX=_pr_456 ./compose run --rm api manage test
+#        ./compose up -e DEV_IMAGE_SUFFIX=_testing
+#
+# or set here to persist for every ./compose call.
+# DEV_IMAGE_SUFFIX=""
+
+# Volume suffix can be individually set
+# defaults to the same as DEV_IMAGE_SUFFIX
+# DEV_VOLUME_SUFFIX=${DEV_IMAGE_SUFFIX:-}
diff --git a/CHANGES/387.misc b/CHANGES/387.misc
@@ -0,0 +1 @@
+Improvements to dev environment make targets and image/volume snapshoting.
diff --git a/Makefile b/Makefile
@@ -1,6 +1,5 @@
 DOCKER_IMAGE_NAME = localhost/galaxy_ng/galaxy_ng
 
-
 .PHONY: help
 help:             ## Show the help.
 	@echo "Usage: make <target>"
@@ -11,6 +10,8 @@ help:             ## Show the help.
 # ANSIBLE_SKIP_CONFLICT_CHECK=1 tells pip/pip-compile to avoid
 # refusing to update ansbile 2.9 to ansible 2.10.
 
+# Repository management
+
 .PHONY: requirements
 requirements:     ## Update python dependencies lock files (i.e. requirements.txt).
 	ANSIBLE_SKIP_CONFLICT_CHECK=1 pip-compile -U -o requirements/requirements.common.txt \
@@ -20,12 +21,60 @@ requirements:     ## Update python dependencies lock files (i.e. requirements.tx
 	ANSIBLE_SKIP_CONFLICT_CHECK=1 pip-compile -U -o requirements/requirements.insights.txt \
 		setup.py requirements/requirements.insights.in
 
+.PHONY: changelog
+changelog:        ## Build the changelog
+	towncrier build
+
+# Container environment management
 
 .PHONY: docker/build
 docker/build:     ## Build all development images.
 	./compose build
 
-
+# e.g: make docker/test TEST=.api.test_api_ui_sync_config
+# if TEST is not passed run all tests.
 .PHONY: docker/test
 docker/test:      ## Run unit tests.
-	./compose run api manage test galaxy_ng.tests.unit
+	./compose run api manage test galaxy_ng.tests.unit$(TEST)
+
+.PHONY: docker/loaddata
+docker/loaddata:  ## Load initial data from fixtures
+	./compose run --rm -e PULP_FIXTURE_DIRS='["/src/galaxy_ng/dev/automation-hub"]' \
+ api manage loaddata initial_data.json
+
+.PHONY: docker/migrate
+docker/migrate:   ## Run django migrations
+	./compose run --rm api manage migrate
+
+.PHONY: docker/resetdb
+docker/resetdb:   ## Cleans database
+	./compose run --rm api /bin/bash -c "./entrypoint.sh manage reset_db && django-admin migrate"
+
+# Application management and debugging
+
+# e.g: make api/get URL=/content/community/v3/collections/
+.PHONY: api/get
+api/get:          ## Make an api get request using 'httpie'
+	# Makes 2 requests: One to get the token and another to request given URL
+	http --version && (http :8002/api/automation-hub/$(URL) "Authorization: Token $$(http --session DEV_SESSION --auth admin:admin -v POST 'http://localhost:5001/api/automation-hub/v3/auth/token/' username=admin password=admin -b | jq -r '.token')" || echo "http error, check if api is running.") || echo "!!! this command requires httpie - please run 'pip install httpie'"
+
+.PHONY: api/shell
+api/shell:        ## Opens django management shell in api container
+	# Tries to exec in a running container or start a containers and run
+	./compose exec api django-admin shell_plus || ./compose run --use-aliases --service-ports --rm api manage shell_plus
+
+.PHONY: api/bash
+api/bash:         ## Opens bash session in the api container
+	# tries to exec in a running container or start a container and run
+	./compose exec api /bin/bash || ./compose run --use-aliases --service-ports --rm api /bin/bash
+
+.PHONY: api/runserver
+api/runserver:    ## Runs api using django webserver for debugging
+	# Stop all running containers if any
+	./compose stop
+	# Start only services if containers exists, else create the containers and start.
+	./compose start worker resource-manager content-app ui || ./compose up -d worker resource-manager content-app ui
+	# ensure API is not running
+	./compose stop api
+	# Run api using django runserver for debugging
+	./compose run --service-ports --use-aliases --name api --rm api manage runserver 0.0.0.0:8000
diff --git a/compose b/compose
@@ -5,7 +5,8 @@ set -o errexit
 
 if [[ -f '.compose.env' ]]; then
   set -o allexport
-  source .compose.env
+  # avoid override already set variables
+  source <(grep -v '^#' .compose.env | sed -E 's|^(.+)=(.*)$|: ${\1=\2}; export \1|g')
   set +o allexport
 fi
 
@@ -20,6 +21,9 @@ EOF
 fi
 
 echo "INFO: Using compose profile ${COMPOSE_PROFILE}"
+echo "INFO: ${DEV_SOURCE_PATH:-No} packages installed from source"
+echo "INFO: Image suffix ${DEV_IMAGE_SUFFIX:-is unset}"
+echo "INFO: Volume suffix ${DEV_VOLUME_SUFFIX:-${DEV_IMAGE_SUFFIX:-is unset}}"
 
 compose_args=(
   -f 'dev/docker-compose.yml'
@@ -40,8 +44,11 @@ else
 fi
 
 declare -xr DEV_SOURCE_PATH=${DEV_SOURCE_PATH:-galaxy_ng}
-declare -xr COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-galaxy_ng}"
 declare -xr COMPOSE_CONTEXT=".."
 declare -xr LOCK_REQUIREMENTS="${LOCK_REQUIREMENTS:-1}"
+declare -xr COMPOSE_PROFILE="${COMPOSE_PROFILE}"
+declare -xr DEV_IMAGE_SUFFIX="${DEV_IMAGE_SUFFIX:-}"
+declare -xr DEV_VOLUME_SUFFIX="${DEV_VOLUME_SUFFIX:-${DEV_IMAGE_SUFFIX}}"
+declare -xr COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-galaxy_ng${DEV_IMAGE_SUFFIX:-}}"
 
 exec docker-compose "${compose_args[@]}" "$@"
diff --git a/dev/Dockerfile.base b/dev/Dockerfile.base
@@ -5,12 +5,19 @@ ARG USER_NAME=galaxy
 ARG USER_GROUP=galaxy
 ARG COMPOSE_PROFILE
 ARG LOCK_REQUIREMENTS
+ARG DEV_SOURCE_PATH
+ARG DEV_IMAGE_SUFFIX
+ARG DEV_VOLUME_SUFFIX
 
 ENV LANG=en_US.UTF-8 \
     PYTHONUNBUFFERED=1 \
     PULP_SETTINGS=/etc/pulp/settings.py \
     DJANGO_SETTINGS_MODULE=pulpcore.app.settings \
-    LOCK_REQUIREMENTS="${LOCK_REQUIREMENTS}"
+    COMPOSE_PROFILE="${COMPOSE_PROFILE}" \
+    LOCK_REQUIREMENTS="${LOCK_REQUIREMENTS}" \
+    DEV_SOURCE_PATH="${DEV_SOURCE_PATH}" \
+    DEV_IMAGE_SUFFIX="${DEV_IMAGE_SUFFIX}" \
+    DEV_VOLUME_SUFFIX="${DEV_VOLUME_SUFFIX}"
 
 RUN set -ex; \
     id --group "${USER_GROUP}" &>/dev/null \
@@ -50,9 +57,13 @@ RUN set -ex; \
 # Install application
 COPY . /app
 
+# When LOCK_REQUIREMENTS is disabled avoid running collectstatic here
+# on that case developer should run collectstatic manually
 RUN set -ex; \
     pip install --no-cache-dir --editable /app \
-    && PULP_CONTENT_ORIGIN=x django-admin collectstatic
+    && pip install -r /app/requirements/requirements.dev.txt \
+    && if [[ "${LOCK_REQUIREMENTS}" -eq "1" ]]; then \
+    PULP_CONTENT_ORIGIN=x django-admin collectstatic; fi
 
 # Finalize installation
 RUN set -ex; \

diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
@@ -8,7 +8,16 @@ services:
       dockerfile: "dev/Dockerfile.base"
       args:
         LOCK_REQUIREMENTS: "${LOCK_REQUIREMENTS}"
-    image: "localhost/galaxy_ng/galaxy_ng:base"
+        COMPOSE_PROFILE: "${COMPOSE_PROFILE}"
+        DEV_SOURCE_PATH: "${DEV_SOURCE_PATH}"
+        DEV_IMAGE_SUFFIX: "${DEV_IMAGE_SUFFIX:-}"
+        DEV_VOLUME_SUFFIX: "${DEV_VOLUME_SUFFIX:-${DEV_IMAGE_SUFFIX:-}}"
+    image: "localhost/galaxy_ng/galaxy_ng:base${DEV_IMAGE_SUFFIX:-}"
+    environment:
+      - "WITH_DEV_INSTALL=1"
+      - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}"
+      - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}"
+      - "COMPOSE_PROFILE=${COMPOSE_PROFILE}"
     entrypoint: "/bin/true"
     tmpfs:
       - "/var/lib/pulp/artifact"
@@ -20,15 +29,17 @@ services:
       context: "${COMPOSE_CONTEXT}"
       dockerfile: "dev/${COMPOSE_PROFILE}/Dockerfile"
       args:
-        LOCK_REQUIREMENTS: "${LOCK_REQUIREMENTS}"
-    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}"
+        DEV_IMAGE_SUFFIX: "${DEV_IMAGE_SUFFIX:-}"
+        DEV_VOLUME_SUFFIX: "${DEV_VOLUME_SUFFIX:-${DEV_IMAGE_SUFFIX:-}}"
+    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}${DEV_IMAGE_SUFFIX:-}"
     command: ['run', 'api-reload']
     ports:
       - "5001:8000"
     environment:
       - "WITH_DEV_INSTALL=1"
       - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}"
       - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}"
+      - "COMPOSE_PROFILE=${COMPOSE_PROFILE}"
     env_file:
       - './common/galaxy_ng.env'
     volumes:
@@ -43,12 +54,13 @@ services:
       - redis
 
   resource-manager:
-    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}"
+    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}${DEV_IMAGE_SUFFIX:-}"
     command: ['run', 'resource-manager']
     environment:
       - "WITH_DEV_INSTALL=1"
       - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}"
       - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}"
+      - "COMPOSE_PROFILE=${COMPOSE_PROFILE}"
     env_file:
       - './common/galaxy_ng.env'
     volumes:
@@ -63,12 +75,13 @@ services:
       - redis
 
   worker:
-    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}"
+    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}${DEV_IMAGE_SUFFIX:-}"
     command: ['run', 'worker']
     environment:
       - "WITH_DEV_INSTALL=1"
       - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}"
       - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}"
+      - "COMPOSE_PROFILE=${COMPOSE_PROFILE}"
     env_file:
       - './common/galaxy_ng.env'
     volumes:
@@ -83,14 +96,15 @@ services:
       - redis
 
   content-app:
-    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}"
+    image: "localhost/galaxy_ng/galaxy_ng:${COMPOSE_PROFILE}${DEV_IMAGE_SUFFIX:-}"
     command: ['run', 'content-app']
     ports:
       - "24816:24816"
     environment:
       - "WITH_DEV_INSTALL=1"
       - "LOCK_REQUIREMENTS=${LOCK_REQUIREMENTS}"
       - "DEV_SOURCE_PATH=${DEV_SOURCE_PATH}"
+      - "COMPOSE_PROFILE=${COMPOSE_PROFILE}"
     env_file:
       - './common/galaxy_ng.env'
     volumes:
@@ -120,6 +134,9 @@ services:
 
 
 volumes:
-  pulp: {}
-  pg_data: {}
-  redis_data: {}
+  pulp:
+    name: pulp${DEV_VOLUME_SUFFIX:-}
+  pg_data:
+    name: pg_data${DEV_VOLUME_SUFFIX:-}
+  redis_data:
+    name: redis_data${DEV_VOLUME_SUFFIX:-}
diff --git a/dev/insights/Dockerfile b/dev/insights/Dockerfile
@@ -1,4 +1,6 @@
-FROM localhost/galaxy_ng/galaxy_ng:base
+ARG DEV_IMAGE_SUFFIX
+
+FROM localhost/galaxy_ng/galaxy_ng:base${DEV_IMAGE_SUFFIX:-}
 
 COPY requirements/requirements.insights.txt /tmp/requirements.insights.txt
 RUN set -ex; \

diff --git a/dev/standalone/Dockerfile b/dev/standalone/Dockerfile
@@ -1,4 +1,6 @@
-FROM localhost/galaxy_ng/galaxy_ng:base
+ARG DEV_IMAGE_SUFFIX
+
+FROM localhost/galaxy_ng/galaxy_ng:base${DEV_IMAGE_SUFFIX:-}
 
 COPY requirements/requirements.standalone.txt /tmp/requirements.standalone.txt
 

diff --git a/requirements/requirements.dev.txt b/requirements/requirements.dev.txt
@@ -0,0 +1,4 @@
+ipython
+ipdb
+pdbr
+django-extensions
diff --git a/setup.py b/setup.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import re
 import tempfile
 import tarfile
 import urllib.request
@@ -65,6 +66,29 @@ def run(self):
     "pulp-container>=2.3.1"
 ]
 
+
+is_on_dev_environment = (
+    "COMPOSE_PROFILE" in os.environ and "DEV_SOURCE_PATH" in os.environ
+    and os.environ.get("LOCK_REQUIREMENTS") == "0"
+)
+if is_on_dev_environment:
+    """
+    To enable the installation of local dependencies e.g: a local fork of
+    pulp_ansible checked out to specific branch/version.
+    The paths listed on DEV_SOURCE_PATH must be unpinned to avoid pip
+    VersionConflict error.
+    ref: https://github.com/ansible/galaxy_ng/wiki/Development-Setup
+         #steps-to-run-dev-environment-with-specific-upstream-branch
+    """
+    DEV_SOURCE_PATH = os.environ.get("DEV_SOURCE_PATH", "").split(":")
+    DEV_SOURCE_PATH += [path.replace("_", "-") for path in DEV_SOURCE_PATH]
+    requirements = [
+        re.sub(r"""(=|^|~|<|>|!)([\S]+)""", "", req)
+        if req.lower().startswith(tuple(DEV_SOURCE_PATH)) else req
+        for req in requirements
+    ]
+    print("Installing with unpinned DEV_SOURCE_PATH requirements", requirements)
+
 package_name = os.environ.get("GALAXY_NG_ALTERNATE_NAME", "galaxy-ng")
 
 with open('galaxy_ng/app/VERSION') as version_file: