[tooling] Update documentation with uv documentation

.dockerignore

@@ -1,3 +1,4 @@
 build/workspace
 *.pyc
 .git
+.venv

Makefile

@@ -3,8 +3,8 @@ USER_GROUP := $(shell id -u):$(shell id -g)
 UPSTREAM_OWNER := $(shell scripts/git/upstream_owner.sh)
 COMMIT := $(shell scripts/git/commit.sh)
 
-BLACK_EXCLUDES := "/interfaces|/venv"
-ISORT_EXCLUDES := "--extend-skip=/interfaces --extend-skip=/venv"
+BLACK_EXCLUDES := "/interfaces|/venv|/.venv"
+ISORT_EXCLUDES := "--extend-skip=/interfaces --extend-skip=/venv --extends-skip=/.venv/"
 
 ifeq ($(OS),Windows_NT)
   detected_OS := Windows
@@ -49,7 +49,6 @@ shell-lint:
 unit-test:
 	cd monitoring && make unit-test
 
-# TODO: Add dependency on requirements.txt after we are sufficiently sure most users won't encounter a circular dependency
 .PHONY: image
 image:
 	cd monitoring && make image
@@ -111,21 +110,3 @@ restart-all: stop-uss-mocks down-locally start-locally start-uss-mocks
 # For local development when restarts of the mock USS are frequently required
 .PHONY: restart-uss-mocks
 restart-uss-mocks: stop-uss-mocks start-uss-mocks
-
-# Legacy target To be run locally whenever a direct dependency has been updated in requirements.in
-.PHONY: update-pinned-dependencies
-update-pinned-dependencies: requirements.txt
-
-define finalize_requirements_txt
-	awk 'BEGIN { RS = ""; FS = "\n" } { gsub("# by the following command:\n#\n#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in\n#\n", "#\n# See requirements.in to update.\n\n"); print }' requirements.txt > requirements.txt.new && mv requirements.txt.new requirements.txt
-	echo The WARNING above is expected: https://github.com/jazzband/pip-tools/issues/2160
-endef
-
-requirements.txt: requirements.in
-	docker container run -u ${USER_GROUP} -v $(CURDIR):/app/monitoring interuss/monitoring pip-compile --cache-dir /tmp/.pip-cache --generate-hashes --output-file=requirements.txt requirements.in
-	$(call finalize_requirements_txt)
-
-.PHONY: upgrade-requirements
-upgrade-requirements:
-	docker container run -u ${USER_GROUP} -v $(CURDIR):/app/monitoring interuss/monitoring pip-compile --cache-dir /tmp/.pip-cache --upgrade --generate-hashes --output-file=requirements.txt requirements.in
-	$(call finalize_requirements_txt)

introduction_to_repository.md

@@ -45,12 +45,27 @@ The [uss_qualifier](monitoring/uss_qualifier/README.md) is a tool for testing /
 
 ### Updating Dependencies
 
-This project pins its dependencies with the help of [pip-tools](https://pypi.org/project/pip-tools/):
+This project manage its dependencies with the help of [uv](https://docs.astral.sh/uv/).
 
-Direct dependencies are declared in `requirements.in`. This file may be modified to update or add a new dependency.
+You may use [uv features to manage them](https://docs.astral.sh/uv/concepts/projects/dependencies/) or directly update the [pyproject.toml](pyproject.toml) file and run `uv lock`.
 
-When `requirements.in` is updated, the pinned dependencies must be updated by running:
+### Updating python version
 
-```bash
-make requirements.txt
-```
+* Update the requiered version in the [pyproject.toml](pyproject.toml) file
+* Update the docker image version in the [Dockerfile](monitoring/Dockerfile).
+* Run `uv lock` to update the lock file.
+
+This procedure also work for the small mini-project for [isort](test/isort) and [repo_hygiene](test/repo_hygiene).
+
+### Upgrade dependencies
+
+Dependencies are locked to the version specified in the [pyproject.toml](pyproject.toml) file and the lock file. To update a package:
+
+* Ensure the correct version is set in the [pyproject.toml](pyproject.toml) file (no constraints is possible).
+* Run `uv lock --upgrade-package <package>`
+
+You may also upgrade all packages with
+
+* `uv lock --upgrade`
+
+This procedure also work for the small mini-project for [isort](test/isort) and [repo_hygiene](test/repo_hygiene).

monitoring/Dockerfile

@@ -9,7 +9,7 @@
 #
 # This image is intended to be built from the repository root context/folder.
 
-FROM python:3.12.6-slim
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
 # Not -alpine because: https://stackoverflow.com/a/58028091/651139
 
 # Install system tools
@@ -19,9 +19,6 @@ FROM python:3.12.6-slim
 #   ca-certificates: Needed to accurately validate TLS connections
 RUN apt-get update --fix-missing && apt-get install -y make openssl curl libgeos-dev gcc g++ && apt-get install ca-certificates
 
-# Deal with https://github.com/docker-library/python/pull/954
-RUN pip install --no-cache-dir setuptools==74.1.2
-
 # Required to build in an ARM environment
 #   gevent: libffi-dev libssl-dev python3-dev build-essential
 #   lxml: libxml2-dev libxslt-dev
@@ -30,10 +27,22 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then \
     apt-get install -y libffi-dev libssl-dev python3-dev build-essential libxml2-dev libxslt-dev pkg-config libhdf5-dev \
 ; fi
 
-# Install the necessary Python packages from requirements.txt
-RUN mkdir -p /app/monitoring
-COPY ./requirements.txt /app/monitoring/requirements.txt
-RUN pip install -r /app/monitoring/requirements.txt && rm -rf __pycache__
+RUN mkdir -p /app/
+
+# Install dependencies
+ENV UV_LINK_MODE=copy
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=./uv.lock,target=/app/uv.lock \
+    --mount=type=bind,source=./pyproject.toml,target=/app/pyproject.toml \
+    cd /app/ && \
+    uv sync --locked --no-install-project --compile-bytecode
+
+# Ensure the cache folder is present and writable by anyone
+# (Some command run with limited privileges)
+RUN mkdir -p /.cache/uv/ && chmod 777 /.cache/uv/
+
+# Start in this folder
+WORKDIR /app/monitoring
 
 # Add core content from repo
 ADD ./interfaces /app/interfaces
@@ -43,14 +52,11 @@ ADD ./monitoring /app/monitoring
 COPY ./monitoring/health_check.sh /app/health_check.sh
 RUN chmod 766 /app/health_check.sh
 
-# Start in this folder
-WORKDIR /app/monitoring
-
 # Additional preparations for uss_qualifier/webapp
 RUN mkdir -p /app/uss-host-files
 
 # This script indicates the status of the container
-HEALTHCHECK --interval=3s CMD sh /app/health_check.sh
+HEALTHCHECK --interval=3s --start-period=15s CMD sh /app/health_check.sh
 
 # Discover `monitoring` module in Python
 ENV PYTHONPATH=/app

monitoring/Makefile

@@ -10,7 +10,7 @@ format:
 unit-test:
 	cd uss_qualifier && make unit_test
 
-image: ../requirements.txt $(shell find . -type f ! -path "*/output/*" ! -name image ! -name *.pyc) $(shell find ../interfaces -type f)
+image: $(shell find . -type f ! -path "*/output/*" ! -name image ! -name *.pyc) $(shell find ../interfaces -type f)
 	# Building image due to changes in the following files: $?
 	./build.sh
 

monitoring/deployment_manager/deploylib/port_forwarding.py

@@ -67,7 +67,7 @@ def init_poolmanager(
         connections,
         maxsize,
         block=requests.adapters.DEFAULT_POOLBLOCK,
-        **pool_kwargs
+        **pool_kwargs,
     ):
         """Overrides method in base class"""
         self.poolmanager = SocketPoolManager(
@@ -114,7 +114,7 @@ def _new_conn(self):
             host=self.host,
             port=self.port,
             timeout=self.timeout.connect_timeout,
-            **self.conn_kw
+            **self.conn_kw,
         )
 
 
@@ -153,7 +153,7 @@ def _new_conn(self):
             timeout=self.timeout.connect_timeout,
             cert_file=self.cert_file,
             key_file=self.key_file,
-            **self.conn_kw
+            **self.conn_kw,
         )
 
         return self._prepare_conn(conn)

monitoring/loadtest/README.md

@@ -8,32 +8,30 @@ In each files every action has a weight declared in the `@task(n)` decorator. Yo
 
 ## Run locally without Docker
 1. Go to the repository's root directory. We have to execute from root directory due to our directory structure choice.
-1. Create virtrual environment `virtualenv --python python3 ./monitoring/loadtest/env`
-1. Activate virtual environment `. ./monitoring/loadtest/env/bin/activate`
-1. Install dependencies `pip install -r ./monitoring/loadtest/requirements.txt`
+1. Install UV: https://docs.astral.sh/uv/getting-started/installation/
 1. Set OAuth Spec with environment variable `AUTH_SPEC`. See [the auth spec documentation](../monitorlib/README.md#Auth_specs)
 for the format of these values.  Omitting this step will result in Client Initialization failure.
 1. You have 2 options of load testing the ISA or Subscription workflow
 
-    a. For ISA run: `AUTH_SPEC="<auth spec>" locust -f ./monitoring/loadtest/locust_files/ISA.py -H <DSS Endpoint URL>`
+    a. For ISA run: `AUTH_SPEC="<auth spec>" uv run locust -f ./monitoring/loadtest/locust_files/ISA.py -H <DSS Endpoint URL>`
 
-    b. For Subscription run: `AUTH_SPEC="<auth spec>" locust -f ./monitoring/loadtest/locust_files/Sub.py -H <DSS Endpoint URL>`
+    b. For Subscription run: `AUTH_SPEC="<auth spec>" uv run locust -f ./monitoring/loadtest/locust_files/Sub.py -H <DSS Endpoint URL>`
 
 ## Running in a Container
 Simply build the Docker container with the Dockerfile from the root directory. All the files are added into the container
 
 1. From the root folder of this repository, build the monitoring image with `make image`
 1. Run Docker container; in general:
 
-    a. For ISA run: `docker run -e AUTH_SPEC="<auth spec>" -p 8089:8089 interuss/monitoring locust -f loadtest/locust_files/ISA.py`
+    a. For ISA run: `docker run -e AUTH_SPEC="<auth spec>" -p 8089:8089 interuss/monitoring uv run locust -f loadtest/locust_files/ISA.py`
 
-    b. For Sub run: `docker run -e AUTH_SPEC="<auth spec>" -p 8089:8089 interuss/monitoring locust -f loadtest/locust_files/Sub.py`
+    b. For Sub run: `docker run -e AUTH_SPEC="<auth spec>" -p 8089:8089 interuss/monitoring uv run locust -f loadtest/locust_files/Sub.py`
 
 1. If testing local DSS instance, be sure that the loadtest (monitoring) container has access to the DSS container:
 
-    a. For ISA run: `docker run -e AUTH_SPEC="DummyOAuth(http://oauth.authority.localutm:8085/token,uss1)" --network="interop_ecosystem_network" -p 8089:8089 interuss/monitoring locust -f loadtest/locust_files/ISA.py`
+    a. For ISA run: `docker run -e AUTH_SPEC="DummyOAuth(http://oauth.authority.localutm:8085/token,uss1)" --network="interop_ecosystem_network" -p 8089:8089 interuss/monitoring uv run locust -f loadtest/locust_files/ISA.py`
 
-    b. For Sub run: `docker run -e AUTH_SPEC="DummyOAuth(http://oauth.authority.localutm:8085/token,uss1)" --network="interop_ecosystem_network" -p 8089:8089 interuss/monitoring locust -f loadtest/locust_files/Sub.py`
+    b. For Sub run: `docker run -e AUTH_SPEC="DummyOAuth(http://oauth.authority.localutm:8085/token,uss1)" --network="interop_ecosystem_network" -p 8089:8089 interuss/monitoring uv run locust -f loadtest/locust_files/Sub.py`
 
 ## Use
 1. Navigate to http://localhost:8089

monitoring/loadtest/locust_files/ISA.py

@@ -10,7 +10,13 @@
 
 from monitoring.monitorlib import rid_v1
 from monitoring.monitorlib.testing import make_fake_url
-from monitoring.prober.rid.v1 import common
+
+VERTICES = [
+    {"lng": 130.6205, "lat": -23.6558},
+    {"lng": 130.6301, "lat": -23.6898},
+    {"lng": 130.6700, "lat": -23.6709},
+    {"lng": 130.6466, "lat": -23.6407},
+]
 
 
 class ISA(client.USS):
@@ -29,7 +35,7 @@ def create_isa(self):
                 "extents": {
                     "spatial_volume": {
                         "footprint": {
-                            "vertices": common.VERTICES,
+                            "vertices": VERTICES,
                         },
                         "altitude_lo": 20,
                         "altitude_hi": 400,
@@ -58,7 +64,7 @@ def update_isa(self):
                 "extents": {
                     "spatial_volume": {
                         "footprint": {
-                            "vertices": common.VERTICES,
+                            "vertices": VERTICES,
                         },
                         "altitude_lo": 20,
                         "altitude_hi": 400,

monitoring/mock_uss/start.sh

@@ -20,7 +20,7 @@ cp health_check.sh /app
 # Start mock_uss server
 port=${MOCK_USS_PORT:-5000}
 export PYTHONUNBUFFERED=TRUE
-gunicorn \
+uv run gunicorn \
     --preload \
     --config ./gunicorn.conf.py \
     --worker-class="gevent" \

monitoring/mock_uss/test.sh

@@ -14,4 +14,4 @@ else
 fi
 cd "${BASEDIR}" || exit 1
 
-pytest -s -vvv
+uv run pytest -s -vvv

monitoring/prober/README.md

@@ -44,7 +44,7 @@ make image
 ```shell script
 docker run --rm interuss/monitoring \
     -w /app/monitoring/prober \
-    pytest \
+    uv run pytest \
     --dss-endpoint <URL> \
     [--rid-auth <SPEC>] \
     [--scd-auth1 <SPEC>] \
@@ -57,17 +57,12 @@ requires more setup and may not be fully-reproducible due to system differences
 in Python, etc.
 
 ### First time set up
-```shell script
-sudo apt-get install virtualenv
-virtualenv --python python3 env
-. ./env/bin/activate
-pip install -r requirements.txt
-```
+
+Install UV: https://docs.astral.sh/uv/getting-started/installation/
 
 ### Running the prober
 ```shell
-. ./env/bin/activate
-pytest \
+uv run pytest \
     --dss-endpoint <URL> \
     [--rid-auth <SPEC>] \
     [--scd-auth1 <SPEC>] \

monitoring/prober/run_locally.sh

@@ -40,7 +40,7 @@ if ! docker run \
 	-v "$(pwd)/$OUTPUT_DIR:/app/$OUTPUT_DIR" \
 	-w /app/monitoring/prober \
 	interuss/monitoring \
-	pytest \
+	uv run pytest \
 	"${1:-.}" \
 	-rsx \
 	--junitxml="/app/$OUTPUT_DIR/e2e_test_result" \

monitoring/uss_qualifier/README.md

@@ -8,7 +8,7 @@ requirements, including interoperability of multiple USS/USSPs.
 
 ## Usage
 
-The `uss_qualifier` tool is a synchronous executable built into the `interuss/monitoring` Docker image.  To use the `interuss/monitoring` image to run uss_qualifier, specify a working directory of `/app/monitoring/uss_qualifier` and a command of `python main.py ${OPTIONS}` -- see [`run_locally.sh`](run_locally.sh) for an example that can be run on any local system that is running the required prerequisites (documented in message printed by run_locally.sh).
+The `uss_qualifier` tool is a synchronous executable built into the `interuss/monitoring` Docker image.  To use the `interuss/monitoring` image to run uss_qualifier, specify a working directory of `/app/monitoring/uss_qualifier` and a command of `uv run main.py ${OPTIONS}` -- see [`run_locally.sh`](run_locally.sh) for an example that can be run on any local system that is running the required prerequisites (documented in message printed by run_locally.sh).
 
 The primary input accepted by uss_qualifier is the "configuration" specified with the `--config` option.  This option should be a [reference to a configuration file](configurations/README.md) that the user has constructed or been provided to test the desired system for the desired characteristics.  If testing a standard local system (DSS + dummy auth + mock USSs), the user can specify an alternate configuration reference as a single argument to `run_locally.sh` (the default configuration is `configurations.dev.local_test`).
 

monitoring/uss_qualifier/bin/generate_rid_test_definition.sh

@@ -46,6 +46,6 @@ docker run ${docker_args} --name uss_qualifier \
   -v "$(pwd):/app" \
   -w /app/monitoring/uss_qualifier \
   interuss/monitoring \
-  python main.py $QUALIFIER_OPTIONS
+  uv run main.py $QUALIFIER_OPTIONS
 
 rm ${CONFIG_LOCATION}

monitoring/uss_qualifier/bin/run.sh

@@ -50,4 +50,4 @@ docker run ${docker_args} --name uss_qualifier \
   -v "${CONFIG_LOCATION}:/config.json" \
   -w /app/monitoring/uss_qualifier \
   interuss/monitoring \
-  python main.py $QUALIFIER_OPTIONS
+  uv run main.py $QUALIFIER_OPTIONS

monitoring/uss_qualifier/make_artifacts.sh

@@ -57,4 +57,4 @@ docker run --name uss_qualifier \
   -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
   -w /app/monitoring/uss_qualifier \
   interuss/monitoring \
-  python make_artifacts.py $MAKE_ARTIFACTS_OPTIONS
+  uv run make_artifacts.py $MAKE_ARTIFACTS_OPTIONS

monitoring/uss_qualifier/run_locally.sh

@@ -83,4 +83,4 @@ docker run ${docker_args} --name uss_qualifier \
   -v "$(pwd)/$CACHE_DIR:/app/$CACHE_DIR" \
   -w /app/monitoring/uss_qualifier \
   interuss/monitoring \
-  python main.py $QUALIFIER_OPTIONS
+  uv run main.py $QUALIFIER_OPTIONS

monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py

@@ -574,7 +574,7 @@ def _assert_generic_evaluator_result(
     *setters_and_values: list[Any],
     outcome: bool,
     wanted_fail: Optional[list[str]] = None,
-    rid_version: Optional[RIDVersion] = None
+    rid_version: Optional[RIDVersion] = None,
 ):
     """
     Helper to call _assert_generic_evaluator_call that build mocked objects first and do the call.
@@ -660,7 +660,7 @@ def _assert_generic_evaluator_correct_field_is_used(
 def _assert_generic_evaluator_valid_value(
     *fct_and_setters: list[Any],
     valid_value: T,
-    rid_version: Optional[RIDVersion] = None
+    rid_version: Optional[RIDVersion] = None,
 ):
     """
     Test that a _evaluate function is handeling a specifc value as valid.
@@ -691,7 +691,7 @@ def _assert_generic_evaluator_invalid_value(
     *fct_and_setters: list[Any],
     invalid_value: T,
     valid_value: T,
-    rid_version: Optional[RIDVersion] = None
+    rid_version: Optional[RIDVersion] = None,
 ):
     """
     Test that a _evaluate function is handeling a specifc value as invalid.
@@ -737,7 +737,7 @@ def _assert_generic_evaluator_invalid_value(
 def _assert_generic_evaluator_invalid_observed_value(
     *fct_and_setters: list[Any],
     invalid_value: T,
-    rid_version: Optional[RIDVersion] = None
+    rid_version: Optional[RIDVersion] = None,
 ):
     """
     Test that a _evaluate function is handeling a specifc value as invalid when observed.

monitoring/uss_qualifier/scripts/in_container/format_test_documentation.sh

@@ -15,4 +15,4 @@ fi
 cd "${BASEDIR}/../.." || exit 1
 
 # Run validation
-python scenarios/documentation/format_documentation.py
+uv run scenarios/documentation/format_documentation.py

monitoring/uss_qualifier/scripts/in_container/format_test_suite_docs.sh

@@ -15,4 +15,4 @@ fi
 cd "${BASEDIR}/../.." || exit 1
 
 # Run validation
-python suites/documentation/format_documentation.py "$@"
+uv run suites/documentation/format_documentation.py "$@"

monitoring/uss_qualifier/scripts/in_container/run_unit_tests.sh

@@ -14,4 +14,4 @@ else
 fi
 cd "${BASEDIR}/../.." || exit 1
 
-pytest
+uv run pytest

monitoring/uss_qualifier/scripts/in_container/validate_test_definitions.sh

@@ -15,4 +15,4 @@ fi
 cd "${BASEDIR}/../.." || exit 1
 
 # Run validation
-python scenarios/documentation/validate_documentation.py
+uv run scenarios/documentation/validate_documentation.py