Modernize and charm 'errors' Django app

.github/copilot-instructions.md

@@ -51,12 +51,10 @@ src/                  # Error Tracker application source
     amqp_utils.py     #     RabbitMQ/AMQP utilities
     config.py         #     Configuration handling
     utils.py          #     Shared utilities
+  Makefile            #   Makefile with targets: run-daisy, run-errors, run-retracer, populate-test-data
   retracer/           #   Symbolic retracer (turns addresses into stack frames)
     config/           #     Per-release retracer configuration
   retracer.py         #   Retracer entry point
-  run-daisy.sh        #   Script to start daisy locally
-  run-errors.sh       #   Script to start errors locally
-  run-retracer.sh     #   Script to start retracer locally
   tools/              #   Maintenance and housekeeping scripts
   tests/              #   Application tests
 
@@ -101,7 +99,7 @@ The application relies on:
 
 ### Test Architecture
 
-There are two layers of testing:
+There are multiple layers of testing:
 
 1. **Unit/functional tests** (`src/tests/`) — pytest-based tests that require Cassandra, RabbitMQ, and Swift running locally. These test individual components (submission, retracing, Cassandra operations, OOPS processing).
 

.gitignore

@@ -66,6 +66,7 @@ cover/
 local_config.py
 db.sqlite3
 db.sqlite3-journal
+src/static
 
 # Flask stuff:
 instance/

README.md

@@ -23,7 +23,7 @@ Start with the Python dependencies
 # For 'daisy' only
 sudo apt install apport-retrace python3-amqp python3-bson python3-cassandra python3-flask python3-mock python3-pygit2 python3-pytest python3-pytest-cov python3-swiftclient ubuntu-dbgsym-keyring
 # Add this for 'errors'
-sudo apt install python3-django-tastypie python3-numpy python3-social-django
+sudo apt install python3-django-tastypie python3-numpy python3-social-django python3-openid-teams
 ```
 
 Then start a local Cassandra, RabbitMQ and swift (`docker` should work fine too):
@@ -48,19 +48,21 @@ Or start each individual process (from the `./src` folder):
 
 daisy:
 ```
-./run-daisy.sh
+make daisy-run
 ```
 
 retracer:
 ```
-./run-retracer.sh
+make retracer-run
 ```
 
 errors:
 ```
-./run-errors.sh
+make errors-run
 ```
 
+If you need test data in Cassandra, you can run `make populate-test-data` in that same folder.
+
 From there, you can manually upload a crash with the following, from any folder
 containing a `.crash` file with its corresponding `.upload` file:
 ```

charm/charm.py

@@ -26,6 +26,12 @@ def __init__(self, *args):
             ports=[self._error_tracker.daisy_port],
             relation_name="route_daisy",
         )
+        self.route_web = HaproxyRouteRequirer(
+            self,
+            service="web",
+            ports=[self._error_tracker.web_port],
+            relation_name="route_web",
+        )
 
         self.framework.observe(self.on.start, self._on_start)
         self.framework.observe(self.on.install, self._on_install)
@@ -62,15 +68,18 @@ def _on_config_changed(self, event: ops.ConfigChangedEvent):
         # This is a bit annoying, but also doesn't have a very big impact in
         # practice. This charm has no configuration where it's supposed to store
         # data, so it's always very easy to remove a unit and recreate.
+        ports = []
         if enable_daisy:
             self._error_tracker.configure_daisy()
-            self.unit.set_ports(self._error_tracker.daisy_port)
+            ports.append(self._error_tracker.daisy_port)
         if enable_retracer:
             self._error_tracker.configure_retracer(self.config.get("retracer_failed_queue"))
         if enable_timers:
             self._error_tracker.configure_timers()
         if enable_web:
             self._error_tracker.configure_web()
+            ports.append(self._error_tracker.web_port)
+        self.unit.set_ports(*ports)
 
         self.unit.set_workload_version(self._error_tracker.get_version())
         self.unit.status = ops.ActiveStatus("Ready")

charm/errortracker.py

@@ -50,6 +50,7 @@ def __init__(self):
         self.enable_daisy = True
         self.enable_web = True
         self.daisy_port = 8000
+        self.web_port = 9000
 
     def install(self):
         self._install_deps()
@@ -238,3 +239,47 @@ def configure_timers(self):
 
     def configure_web(self):
         logger.info("Configuring web")
+        logger.info("Installing additional web dependencies")
+        check_call(
+            [
+                "apt-get",
+                "install",
+                "-y",
+                "python3-django",
+                "python3-django-tastypie",
+                "python3-numpy",
+                "python3-openid-teams",
+                "python3-social-django",
+                "python3-uwsgidecorators",
+                "uwsgi-plugin-python3",
+            ]
+        )
+        systemd_unit_location = Path("/") / "etc" / "systemd" / "system"
+        systemd_unit_location.mkdir(parents=True, exist_ok=True)
+        (systemd_unit_location / "et-web.service").write_text(
+            f"""
+[Unit]
+Description=Error Tracker web
+After=network.target
+
+[Service]
+User=ubuntu
+Group=ubuntu
+WorkingDirectory={REPO_LOCATION}/src
+ExecStartPre={REPO_LOCATION}/src/errors/manage.py migrate --no-input
+ExecStartPre={REPO_LOCATION}/src/errors/manage.py collectstatic --no-input --clear
+ExecStart=bash -c 'exec uwsgi --plugins python3 --http-socket 0.0.0.0:{self.web_port} --wsgi-file {REPO_LOCATION}/src/errors/wsgi.py --static-map "/static={REPO_LOCATION}/src/static/" --chdir {REPO_LOCATION}/src/ --die-on-term --master --env PYTHONPATH={REPO_LOCATION}/src/ --max-requests 4000 --max-worker-lifetime 21600 --processes "$(($(nproc) * 2))"'
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+"""
+        )
+
+        check_call(["systemctl", "daemon-reload"])
+
+        logger.info("enabling systemd units")
+        check_call(["systemctl", "enable", "et-web"])
+
+        logger.info("restarting systemd units")
+        check_call(["systemctl", "restart", "et-web"])

charm/tests/integration/test_daisy.py

@@ -37,15 +37,15 @@ def test_deploy(
 
 
 def test_http(juju: jubilant.Juju):
-    juju.deploy(HAPROXY, channel="2.8/edge", config={"external-hostname": "daisy.internal"})
+    external_hostname = "daisy.internal"
+    juju.deploy(HAPROXY, channel="2.8/edge", config={"external-hostname": external_hostname})
     juju.deploy(SSC, channel="1/edge")
 
     juju.integrate(HAPROXY + ":certificates", SSC + ":certificates")
     juju.integrate("daisy:route_daisy", HAPROXY)
     juju.wait(lambda status: jubilant.all_active(status, HAPROXY, SSC), timeout=1800)
 
     haproxy_ip = juju.status().apps[HAPROXY].units[f"{HAPROXY}/0"].public_address
-    external_hostname = "daisy.internal"
 
     session = Session()
     session.mount("https://", DNSResolverHTTPSAdapter(external_hostname, haproxy_ip))

charm/tests/integration/test_web.py

@@ -0,0 +1,68 @@
+import logging
+
+import jubilant
+from requests import Session
+from tenacity import Retrying, stop_after_attempt, wait_exponential
+from utils import DNSResolverHTTPSAdapter, check_config
+
+logger = logging.getLogger()
+
+HAPROXY = "haproxy"
+SSC = "self-signed-certificates"
+
+
+def test_deploy(
+    juju: jubilant.Juju,
+    amqp: dict[str, str],
+    cassandra: dict[str, str],
+    swift: dict[str, str],
+    error_tracker_config: str,
+    charm_path: str,
+):
+    juju.deploy(
+        charm=charm_path,
+        app="web",
+        config={
+            "configuration": error_tracker_config,
+            "enable_daisy": False,
+            "enable_retracer": False,
+            "enable_timers": False,
+            "enable_web": True,
+        },
+    )
+
+    juju.wait(lambda status: jubilant.all_active(status, "web"), timeout=600)
+
+    check_config(juju, amqp, cassandra, swift, "web/0")
+
+
+def test_http(juju: jubilant.Juju):
+    external_hostname = "errors.internal"
+    juju.deploy(HAPROXY, channel="2.8/edge", config={"external-hostname": external_hostname})
+    juju.deploy(SSC, channel="1/edge")
+
+    juju.integrate(HAPROXY + ":certificates", SSC + ":certificates")
+    juju.integrate("web:route_web", HAPROXY)
+    juju.wait(lambda status: jubilant.all_active(status, HAPROXY, SSC), timeout=1800)
+
+    haproxy_ip = juju.status().apps[HAPROXY].units[f"{HAPROXY}/0"].public_address
+
+    session = Session()
+    session.mount("https://", DNSResolverHTTPSAdapter(external_hostname, haproxy_ip))
+
+    # Let give this test a few chances to succeed, as it can sometimes be a bit
+    # early and hit 503
+    for attempt in Retrying(
+        stop=stop_after_attempt(10),
+        wait=wait_exponential(min=5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            response = session.get(
+                f"https://{haproxy_ip}/",
+                headers={"Host": external_hostname},
+                verify=False,
+                timeout=30,
+            )
+            assert response.status_code == 200
+            assert "We collect hundreds of thousands of error reports daily" in response.text

charmcraft.yaml

@@ -87,3 +87,7 @@ requires:
     interface: haproxy-route
     limit: 1
     optional: true
+  route_web:
+    interface: haproxy-route
+    limit: 1
+    optional: true

src/Makefile

@@ -0,0 +1,25 @@
+BASE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+export PYTHONPATH := $(BASE_DIR)
+
+.PHONY: daisy-run errors-run errors-shell retracer-run populate-test-data
+
+daisy-run:
+	python3 $(BASE_DIR)daisy/app.py
+
+errors-run:
+	python3 $(BASE_DIR)errors/manage.py migrate
+	python3 $(BASE_DIR)errors/manage.py runserver
+
+errors-uwsgi:
+	python3 $(BASE_DIR)errors/manage.py migrate
+	uwsgi --plugins python3 --http-socket 0.0.0.0:9000 --wsgi-file $(BASE_DIR)errors/wsgi.py --static-map "/static=$(BASE_DIR)../static/" --chdir $(BASE_DIR) --die-on-term --master --env PYTHONPATH=$(BASE_DIR)
+
+errors-shell:
+	python3 $(BASE_DIR)errors/manage.py shell
+
+retracer-run:
+	python3 $(BASE_DIR)retracer.py -a amd64 --sandbox-dir /tmp/sandbox -v --config-dir $(BASE_DIR)retracer/config
+
+populate-test-data:
+	python3 $(BASE_DIR)tests/create_test_data.py