Improve debugging tools for ray and segmentation faults (#245)

* testing branches for scos-actions and scos-tekrsa * record Ray reference creation sites in debug mode * start ray dashboard in debug mode * use faulthandler in debug mode * Compile deps * remove ray init from scheduler * decrease ray version * Update Ray to 2.4.0 * tagged versions of actions and tekrsa * increase sleep time before CI container health check * try 60s wait for API container test * do not use grep quiet option using "-q", grep always has an exit status of 0. Without this flag, exit status is only 0 if a line is selected by grep * update pre-commit hooks * run pre-commit hooks on all files * use sea-prototype-v0.4.1 tags * recompile requirements * update markdownlint hook * revert unnecessary GH actions workflow changes * Bump minimum ray debug version to match scos-actions * Increase wait for container healthy time in GH Action * update scos-actions branch for testing * update scos-tekrsa version for testing * update requirements tagged scos plugin versions, other minor updates
NTIA · Jun 30, 2023 · eb4be4d · eb4be4d
1 parent f2de439
commit eb4be4d
Show file tree

Hide file tree

Showing 19 changed files with 111 additions and 36 deletions.
diff --git a/.github/workflows/github-actions-test.yml b/.github/workflows/github-actions-test.yml
@@ -77,7 +77,7 @@ jobs:
           docker-compose up -d
 
       - name: Wait for containers # wait for containers to finish starting
-        run: sleep 30
+        run: sleep 45
 
       - name: Check API container
         run: docker ps | grep api | grep -q healthy
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,10 +19,10 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.4.0
     hooks:
       - id: pyupgrade
-        args: ["--py3-plus"]
+        args: ["--py38-plus"]
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
@@ -31,12 +31,12 @@ repos:
         types: [file, python]
         args: ["--profile", "black", "--filter-files", "--gitignore"]
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
         types: [file, python]
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.33.0
+    rev: v0.35.0
     hooks:
       - id: markdownlint
         types: [file, markdown]

diff --git a/env.template b/env.template
@@ -48,6 +48,7 @@ SENTRY_DSN=
 
 if $DEBUG; then
     GUNICORN_LOG_LEVEL=debug
+    RAY_record_ref_creation_sites=1
 else
     GUNICORN_LOG_LEVEL=info
 fi

diff --git a/src/authentication/management/commands/get_auth_token.py b/src/authentication/management/commands/get_auth_token.py
@@ -17,5 +17,5 @@ def handle(self, *args, **options):
             self.stdout.write(key)
         else:
             self.stdout.write(
-                "No user with username={} found in the database.".format(username)
+                f"No user with username={username} found in the database."
             )
diff --git a/src/handlers/measurement_handler.py b/src/handlers/measurement_handler.py
@@ -56,4 +56,4 @@ def measurement_action_completed_callback(sender, **kwargs):
         acquisition.data_encrypted = False
     acquisition.save()
 
-    logger.debug("Saved new file at {}".format(acquisition.data.path))
+    logger.debug(f"Saved new file at {acquisition.data.path}")
diff --git a/src/requirements-dev.in b/src/requirements-dev.in
@@ -3,4 +3,5 @@
 pre-commit>=3.0, <4.0
 pytest-cov>=3.0, <4.0
 pytest-django>=4.0, <5.0
+ray[default]>=2.5.0
 tox>=4.0,<5.0
diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt
@@ -4,26 +4,40 @@
 #
 #    pip-compile requirements-dev.in
 #
+aiohttp==3.8.4
+    # via
+    #   aiohttp-cors
+    #   ray
+aiohttp-cors==0.7.0
+    # via ray
 aiosignal==1.3.1
     # via
     #   -r requirements.txt
+    #   aiohttp
     #   ray
 asgiref==3.6.0
     # via
     #   -r requirements.txt
     #   django
+async-timeout==4.0.2
+    # via aiohttp
 attrs==22.2.0
     # via
     #   -r requirements.txt
+    #   aiohttp
     #   jsonschema
     #   pytest
     #   ray
 bcrypt==4.0.1
     # via
     #   -r requirements.txt
     #   paramiko
+blessed==1.20.0
+    # via gpustat
 cachetools==5.3.0
-    # via tox
+    # via
+    #   google-auth
+    #   tox
 certifi==2022.12.7
     # via
     #   -r requirements.txt
@@ -40,13 +54,16 @@ chardet==5.1.0
 charset-normalizer==3.0.1
     # via
     #   -r requirements.txt
+    #   aiohttp
     #   requests
 click==8.1.3
     # via
     #   -r requirements.txt
     #   ray
 colorama==0.4.6
     # via tox
+colorful==0.5.5
+    # via ray
 coreapi==2.3.3
     # via
     #   -r requirements.txt
@@ -118,8 +135,17 @@ filelock==3.9.0
 frozenlist==1.3.3
     # via
     #   -r requirements.txt
+    #   aiohttp
     #   aiosignal
     #   ray
+google-api-core==2.11.0
+    # via opencensus
+google-auth==2.17.3
+    # via google-api-core
+googleapis-common-protos==1.59.0
+    # via google-api-core
+gpustat==1.1
+    # via ray
 grpcio==1.51.3
     # via
     #   -r requirements.txt
@@ -132,6 +158,7 @@ idna==3.4
     # via
     #   -r requirements.txt
     #   requests
+    #   yarl
 inflection==0.5.1
     # via
     #   -r requirements.txt
@@ -165,14 +192,18 @@ marshmallow==3.19.0
     # via
     #   -r requirements.txt
     #   environs
-msgpack==1.0.4
+msgpack==1.0.5
     # via
     #   -r requirements.txt
     #   ray
 msgspec==0.16.0
     # via
     #   -r requirements.txt
     #   scos-actions
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
 nodeenv==1.7.0
     # via pre-commit
 numexpr==2.8.4
@@ -188,10 +219,16 @@ numpy==1.24.2
     #   scos-actions
     #   sigmf
     #   tekrsa-api-wrap
+nvidia-ml-py==11.525.112
+    # via gpustat
 oauthlib==3.2.2
     # via
     #   -r requirements.txt
     #   requests-oauthlib
+opencensus==0.11.2
+    # via ray
+opencensus-context==0.1.3
+    # via opencensus
 packaging==23.0
     # via
     #   -r requirements.txt
@@ -216,20 +253,35 @@ pluggy==1.0.0
     #   tox
 pre-commit==3.3.3
     # via -r requirements-dev.in
-protobuf==4.22.0
+prometheus-client==0.13.1
+    # via ray
+protobuf==4.23.3
     # via
     #   -r requirements.txt
+    #   google-api-core
+    #   googleapis-common-protos
     #   ray
-psutil==5.9.4
+psutil==5.9.5
     # via
     #   -r requirements.txt
+    #   gpustat
     #   scos-actions
 psycopg2-binary==2.9.5
     # via -r requirements.txt
+py-spy==0.3.14
+    # via ray
+pyasn1==0.5.0
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.3.0
+    # via google-auth
 pycparser==2.21
     # via
     #   -r requirements.txt
     #   cffi
+pydantic==1.10.7
+    # via ray
 pyjwt==2.6.0
     # via -r requirements.txt
 pynacl==1.5.0
@@ -271,8 +323,9 @@ pyyaml==5.4.1
     #   docker-compose
     #   pre-commit
     #   ray
-ray==2.5.0
+ray[default]==2.5.1
     # via
+    #   -r requirements-dev.in
     #   -r requirements.txt
     #   scos-actions
 requests==2.31.0
@@ -281,6 +334,7 @@ requests==2.31.0
     #   coreapi
     #   docker
     #   docker-compose
+    #   google-api-core
     #   its-preselector
     #   ray
     #   requests-mock
@@ -289,6 +343,8 @@ requests-mock==1.10.0
     # via -r requirements.txt
 requests-oauthlib==1.3.1
     # via -r requirements.txt
+rsa==4.9
+    # via google-auth
 ruamel-yaml==0.17.21
     # via
     #   -r requirements.txt
@@ -302,11 +358,11 @@ scipy==1.10.1
     # via
     #   -r requirements.txt
     #   scos-actions
-scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0
+scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.1
     # via
     #   -r requirements.txt
     #   scos-tekrsa
-scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0
+scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.1
     # via -r requirements.txt
 sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive
     # via
@@ -315,13 +371,17 @@ sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive
 six==1.16.0
     # via
     #   -r requirements.txt
+    #   blessed
     #   django-session-timeout
     #   dockerpty
+    #   google-auth
     #   jsonschema
     #   python-dateutil
     #   requests-mock
     #   sigmf
     #   websocket-client
+smart-open==6.3.0
+    # via ray
 sqlparse==0.4.4
     # via
     #   -r requirements.txt
@@ -342,6 +402,8 @@ tomli==2.0.1
     #   tox
 tox==4.4.6
     # via -r requirements-dev.in
+typing-extensions==4.5.0
+    # via pydantic
 uritemplate==4.1.1
     # via
     #   -r requirements.txt
@@ -355,12 +417,17 @@ urllib3==1.26.14
 virtualenv==20.20.0
     # via
     #   pre-commit
+    #   ray
     #   tox
+wcwidth==0.2.6
+    # via blessed
 websocket-client==0.59.0
     # via
     #   -r requirements.txt
     #   docker
     #   docker-compose
+yarl==1.9.2
+    # via aiohttp
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/src/requirements.in b/src/requirements.in
@@ -13,8 +13,8 @@ psycopg2-binary>=2.0, <3.0
 pyjwt>=2.4.0, <3.0
 requests-mock>=1.0, <2.0
 requests_oauthlib>=1.0, <2.0
-scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0
-scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0
+scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.1
+scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.1
 
 # The following are sub-dependencies for which SCOS Sensor enforces a
 # higher minimum patch version than the dependencies which require them.

diff --git a/src/requirements.txt b/src/requirements.txt
@@ -99,7 +99,7 @@ markupsafe==2.1.2
     # via jinja2
 marshmallow==3.19.0
     # via environs
-msgpack==1.0.4
+msgpack==1.0.5
     # via ray
 msgspec==0.16.0
     # via scos-actions
@@ -126,9 +126,9 @@ packaging==23.0
     #   ray
 paramiko==3.0.0
     # via docker
-protobuf==4.22.0
+protobuf==4.23.3
     # via ray
-psutil==5.9.4
+psutil==5.9.5
     # via scos-actions
 psycopg2-binary==2.9.5
     # via -r requirements.in
@@ -155,7 +155,7 @@ pyyaml==5.4.1
     # via
     #   docker-compose
     #   ray
-ray==2.5.0
+ray==2.5.1
     # via scos-actions
 requests==2.31.0
     # via
@@ -178,11 +178,11 @@ ruamel-yaml-clib==0.2.7
     # via ruamel-yaml
 scipy==1.10.1
     # via scos-actions
-scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.0
+scos_actions @ git+https://github.com/NTIA/scos-actions@6.3.1
     # via
     #   -r requirements.in
     #   scos-tekrsa
-scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.0
+scos_tekrsa @ git+https://github.com/NTIA/scos-tekrsa@3.1.1
     # via -r requirements.in
 sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive
     # via scos-actions

diff --git a/src/schedule/__init__.py b/src/schedule/__init__.py
@@ -10,7 +10,7 @@ def get_action_with_summary(action):
     summary = get_summary(action_fn)
     action_with_summary = action
     if summary:
-        action_with_summary += " - {}".format(summary)
+        action_with_summary += f" - {summary}"
 
     return action_with_summary
 

diff --git a/src/schedule/models/request.py b/src/schedule/models/request.py
@@ -15,7 +15,7 @@ class Request(models.Model):
 
     def build_absolute_uri(self, location=None):
         """Called from within Django reverse."""
-        scheme_host = "{}://{}".format(self.scheme, self.host)
+        scheme_host = f"{self.scheme}://{self.host}"
         return iri_to_uri(scheme_host + location)
 
     def from_drf_request(self, request, commit=True):

diff --git a/src/schedule/serializers.py b/src/schedule/serializers.py
@@ -14,7 +14,7 @@
 from .models import DEFAULT_PRIORITY, ScheduleEntry
 
 action_help = "[Required] The name of the action to be scheduled"
-priority_help = "Lower number is higher priority (default={})".format(DEFAULT_PRIORITY)
+priority_help = f"Lower number is higher priority (default={DEFAULT_PRIORITY})"
 CHOICES = []
 actions = sorted(registered_actions.keys())
 for action in actions: