bug-1866796: update systemtests (#2885)

Because:
  - Happy path download system tests failed due to stale, hardcoded symbols files in `data/sym_files_to_download.csv`.
  - Even when they weren't stale, these files were from prod and were only available in that env, but we need to be able to run system tests against all envs.
  - The listed symbols files didn't represent all storage buckets (try, regular) we expect to see. Only regular bucket files were tested.

This commit:
  - Adds a new bin/setup-download-tests.py script to setup files for the download test.
  - This script:
    - Fetches symbols files from prod, downloads them, and saves them to two zip files (one for regular files and one for try files) in systemtests/data/zip-files/.
    - Creates a CSV of symbols files from each bucket type (try and regular).
  - Modifies upload and download system tests to upload/download to the correct bucket (try or regular).
  - Rewrites all systemtests/*.sh scripts to *.py for easier debugging.

Notes:
- The zip files will get uploaded to the respective nonprod env's storage backend as part of the upload tests.
- The CSV file is passed to the download tests as before.

This means that now, in any environment, we will have the symbols files listed in the above CSV file uploaded to that environment's storage backend, they will consist of files from both bucket types (try and regular), and they will never be stale.

.gitignore

@@ -23,5 +23,4 @@ yarn-debug.log*
 yarn-error.log*
 
 # Ephemeral systemtest data
-systemtests/data/zip-files/
-systemtests/data/stacks/
+systemtests/data/

docker/config/local_dev.env

@@ -8,7 +8,7 @@
 # ------
 
 # Gunicorn things
-GUNICORN_TIMEOUT=60
+GUNICORN_TIMEOUT=180
 
 # Statsd things
 STATSD_HOST=statsd

docker/config/slick.sh-dist

@@ -18,6 +18,7 @@ FAKEEMAIL=""
 
 # 32-character hex token
 FAKETOKEN="c7c1f8cab79545b6a06bc4122f0eb3cb"
+FAKETOKEN_TRY="ebc447f574924b099e1be15527b62436"
 
 # Build Tecken
 make build
@@ -32,6 +33,7 @@ make setup
 docker compose exec oidcprovider /code/manage.py createuser "${FAKEUSERNAME}" "${FAKEPASSWORD}" "${FAKEEMAIL}"
 docker compose run --rm web bash python manage.py superuser "${FAKEEMAIL}"
 docker compose run --rm web bash python manage.py createtoken "${FAKEEMAIL}" "${FAKETOKEN}"
+docker compose run --rm web bash python manage.py createtoken "--try-upload" "${FAKEEMAIL}" "${FAKETOKEN_TRY}"
 #
 # Reset fakesentry
 curl -X POST http://localhost:8090/api/flush/

docs/dev.rst

@@ -177,7 +177,7 @@ None of these matter except that you need them to enter values into the SSO
 emulation service when you log into your Tecken local dev environment.
 
 You can use ``slick.sh`` to recreate your local dev environment, create a
-superuser account, and create an API token. This simplifies setting everything
+superuser account, and create API tokens. This simplifies setting everything
 up when you're switching contexts or testing things.
 
 .. code-block:: shell

systemtests/README.rst

@@ -18,27 +18,33 @@ Set up the tests this way::
 
     $ make shell
     root@e62fb7ae586f:/app# cd systemtests
-    root@e62fb7ae586f:/app/systemtests# ./setup_tests.sh
+    root@e62fb7ae586f:/app/systemtests# ./setup_tests.py
 
 That creates files in directories under ``data/``.
 
+You only need to set up the tests once to run the system tests against all environments.
+
 
 Running tests
 =============
 
-First, make sure you have a valid, unexpired API token for the environment you're testing.
+First, make sure you have valid, unexpired API tokens for the environment you're testing.
+
+For destructive tests run in local and stage, you need separate auth tokens for try uploads with "Upload Try Symbols Files" permissions. See Bug 1759740.
 
 To set auth tokens, add these to your .env file:
 
 * `LOCAL_AUTH_TOKEN`
+* `LOCAL_AUTH_TOKEN_TRY`
 * `STAGE_AUTH_TOKEN`
+* `STAGE_AUTH_TOKEN_TRY`
 * `PROD_AUTH_TOKEN`
 
 To run the systemtests, do::
 
    $ make shell
    root@f09b3cdf8570:/app# cd systemtests/
-   root@e62fb7ae586f:/app/systemtests# ./test_env.sh ENVIRONMENT
+   root@e62fb7ae586f:/app/systemtests# ./test_env.py ENVIRONMENT
 
 where ``ENVIRONMENT`` is one of the following:
 

systemtests/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.

systemtests/bin/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.

systemtests/bin/download-sym-files.py → systemtests/bin/download_sym_files.py

@@ -67,8 +67,12 @@ def download_sym_files(base_url, csv_file):
         with METRICS.timer("download_time"):
             parts = line.split(",")
 
+            sym_filename, expected_status_code, bucket = parts
+
             # Compute url
-            url = urljoin(base_url, parts[0])
+            url = urljoin(base_url, sym_filename)
+            if bucket == "try":
+                url = url + "?try"
             click.echo(click.style(f"Working on {url} ...", fg="yellow"))
 
             # Download the file
@@ -82,13 +86,15 @@ def download_sym_files(base_url, csv_file):
             click.echo(f">>> status code: {resp.status_code}")
 
             # Compare status code with expected status code
-            if resp.status_code != int(parts[1]):
+            if resp.status_code != int(expected_status_code):
                 click.echo(
                     click.style(
-                        f"FAIL: Status code: {resp.status_code} != {parts[1]}",
+                        f"FAIL: Status code: {resp.status_code} != {expected_status_code}",
                         fg="red",
                     )
                 )
+            else:
+                click.echo(click.style("Success!", fg="green"))
 
 
 if __name__ == "__main__":

systemtests/bin/list-firefox-symbols-zips.py → systemtests/bin/list_firefox_symbols_zips.py

@@ -108,7 +108,8 @@ def get_content_length(url):
     type=int,
     help="max size of files in bytes for urls to print out",
 )
-def run(number, max_size):
+def list_firefox_symbols_zips(number, max_size):
+    urls = []
     for url in get_symbols_urls():
         if number <= 0:
             break
@@ -124,8 +125,11 @@ def run(number, max_size):
             continue
 
         click.echo(url)
+        urls.append(url)
         number -= 1
 
+    return urls
+
 
 if __name__ == "__main__":
-    run()
+    list_firefox_symbols_zips()