Merge pull request #35 from dimagi/sk/record-modules

performance improvement to fetching app data form CC

.env_template

@@ -1,4 +1,5 @@
 CELERY_BROKER_URL=redis://localhost:6379/0
+REDIS_CACHE_URL=redis://localhost:6379/0
 DATABASE_URL=postgres://postgres:postgres@localhost:5432/commcare_connect
 DJANGO_ALLOWED_HOSTS=
 CSRF_TRUSTED_ORIGINS=

.github/workflows/ci.yml

@@ -50,6 +50,7 @@ jobs:
 
     env:
       CELERY_BROKER_URL: 'redis://localhost:6379/0'
+      REDIS_CACHE_URL: 'redis://localhost:6379/0'
       # postgres://user:password@host:port/database
       DATABASE_URL: 'postgres://postgres:postgres@localhost:5432/postgres'
 

commcare_connect/cache.py

@@ -0,0 +1,3 @@
+from quickcache.django_quickcache import get_django_quickcache
+
+quickcache = get_django_quickcache(memoize_timeout=0, timeout=60)

commcare_connect/commcarehq_provider/views.py

@@ -1,4 +1,4 @@
-import requests
+import httpx
 from allauth.socialaccount.providers.oauth2.views import OAuth2Adapter, OAuth2CallbackView, OAuth2LoginView
 from django.conf import settings
 
@@ -14,7 +14,7 @@ class CommcareHQOAuth2Adapter(OAuth2Adapter):
     redirect_uri_protocol = "https"
 
     def complete_login(self, request, app, token, **kwargs):
-        response = requests.get(self.profile_url, headers={"Authorization": f"Bearer {token}"})
+        response = httpx.get(self.profile_url, headers={"Authorization": f"Bearer {token}"})
         extra_data = response.json()
         return self.get_provider().sociallogin_from_response(request, extra_data)
 

commcare_connect/opportunity/forms.py

@@ -100,10 +100,10 @@ def save(self, commit=True):
             if app["id"] == self.cleaned_data["learn_app"]:
                 self.instance.learn_app, _ = CommCareApp.objects.get_or_create(
                     cc_app_id=app["id"],
-                    name=app["name"],
                     cc_domain=app["domain"],
                     organization=organization,
                     defaults={
+                        "name": app["name"],
                         "created_by": self.user.email,
                         "modified_by": self.user.email,
                         "description": self.cleaned_data["learn_app_description"],
@@ -114,10 +114,10 @@ def save(self, commit=True):
             if app["id"] == self.cleaned_data["deliver_app"]:
                 self.instance.deliver_app, _ = CommCareApp.objects.get_or_create(
                     cc_app_id=app["id"],
-                    name=app["name"],
                     cc_domain=app["domain"],
                     organization=organization,
                     defaults={
+                        "name": app["name"],
                         "created_by": self.user.email,
                         "modified_by": self.user.email,
                     },

commcare_connect/utils/commcarehq_api.py

@@ -1,20 +1,25 @@
+import asyncio
 import datetime
+import itertools
 
-import requests
+import httpx
 from allauth.socialaccount.models import SocialAccount, SocialApp, SocialToken
+from asgiref.sync import async_to_sync
 from django.conf import settings
 from django.utils import timezone
 
+from commcare_connect.cache import quickcache
 
-def refresh_access_token(user):
+
+def refresh_access_token(user, force=False):
     social_app = SocialApp.objects.filter(provider="commcarehq").first()
     social_acc = SocialAccount.objects.filter(user=user).first()
     social_token = SocialToken.objects.filter(account=social_acc).first()
 
-    if social_token.expires_at > timezone.now():
+    if not force and social_token.expires_at > timezone.now():
         return social_token
 
-    response = requests.post(
+    response = httpx.post(
         f"{settings.COMMCARE_HQ_URL}/oauth/token/",
         data={
             "grant_type": "refresh_token",
@@ -23,8 +28,10 @@ def refresh_access_token(user):
             "refresh_token": social_token.token_secret,
         },
     )
-    data = response.json()
+    if response.status_code != 200:
+        raise Exception(f"Failed to refresh token: {response.text}")
 
+    data = response.json()
     if data.get("access_token", ""):
         social_token.token = data["access_token"]
         social_token.token_secret = data["refresh_token"]
@@ -34,9 +41,10 @@ def refresh_access_token(user):
     return social_token
 
 
+@quickcache(["user.pk"], timeout=60 * 60)
 def get_domains_for_user(user):
     social_token = refresh_access_token(user)
-    response = requests.get(
+    response = httpx.get(
         f"{settings.COMMCARE_HQ_URL}/api/v0.5/user_domains/",
         headers={"Authorization": f"Bearer {social_token}"},
     )
@@ -45,18 +53,36 @@ def get_domains_for_user(user):
     return domains
 
 
+@quickcache(["user.pk"], timeout=60 * 60)
 def get_applications_for_user(user):
     social_token = refresh_access_token(user)
     domains = get_domains_for_user(user)
+    return _get_applications_for_domains(social_token, domains)
+
+
+@async_to_sync
+async def _get_applications_for_domains(social_token, domains):
+    async with httpx.AsyncClient(timeout=30, headers={"Authorization": f"Bearer {social_token}"}) as client:
+        tasks = []
+        for domain in domains:
+            tasks.append(asyncio.ensure_future(_get_commcare_app_json(client, domain)))
+
+        domain_apps = await asyncio.gather(*tasks)
+    applications = list(itertools.chain.from_iterable(domain_apps))
+    return applications
+
+
+async def _get_commcare_app_json(client, domain):
     applications = []
+    response = await client.get(f"{settings.COMMCARE_HQ_URL}/a/{domain}/api/v0.5/application/")
+    data = response.json()
 
-    for domain in domains:
-        response = requests.get(
-            f"{settings.COMMCARE_HQ_URL}/a/{domain}/api/v0.5/application/",
-            headers={"Authorization": f"Bearer {social_token}"},
-        )
-        data = response.json()
-        for application in data.get("objects", []):
-            applications.append({"id": application.get("id"), "name": application.get("name"), "domain": domain})
+    def _get_name(block: dict):
+        name_data = block.get("name", {})
+        for lang in ["en"] + list(name_data):
+            if lang in name_data:
+                return name_data[lang]
 
+    for application in data.get("objects", []):
+        applications.append({"id": application.get("id"), "name": application.get("name"), "domain": domain})
     return applications

config/settings/base.py

@@ -336,6 +336,20 @@
     "VERSION": "1.0.0",
     "SERVE_PERMISSIONS": ["rest_framework.permissions.IsAdminUser"],
 }
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": env("REDIS_CACHE_URL"),
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            # Mimicing memcache behavior.
+            # https://github.com/jazzband/django-redis#memcached-exceptions-behavior
+            "IGNORE_EXCEPTIONS": True,
+        },
+    }
+}
+
 # ------------------------------------------------------------------------------
 # CommCare Connect Settings...
 # ------------------------------------------------------------------------------

config/settings/local.py

@@ -14,16 +14,6 @@
 ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"] + env.list("DJANGO_ALLOWED_HOSTS", default=[])
 CSRF_TRUSTED_ORIGINS = ["https://*.127.0.0.1", "https://*.loca.lt"] + env.list("CSRF_TRUSTED_ORIGINS", default=[])
 
-# CACHES
-# ------------------------------------------------------------------------------
-# https://docs.djangoproject.com/en/dev/ref/settings/#caches
-CACHES = {
-    "default": {
-        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
-        "LOCATION": "",
-    }
-}
-
 # EMAIL
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend

config/settings/production.py

@@ -20,21 +20,6 @@
 # ------------------------------------------------------------------------------
 DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60)  # noqa: F405
 
-# CACHES
-# ------------------------------------------------------------------------------
-CACHES = {
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": env("REDIS_URL"),
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            # Mimicing memcache behavior.
-            # https://github.com/jazzband/django-redis#memcached-exceptions-behavior
-            "IGNORE_EXCEPTIONS": True,
-        },
-    }
-}
-
 # SECURITY
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header

config/settings/staging.py

@@ -33,21 +33,6 @@
 
 DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60)  # noqa: F405
 
-# CACHES
-# ------------------------------------------------------------------------------
-CACHES = {
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": env("REDIS_URL"),
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            # Mimicing memcache behavior.
-            # https://github.com/jazzband/django-redis#memcached-exceptions-behavior
-            "IGNORE_EXCEPTIONS": True,
-        },
-    }
-}
-
 # SECURITY
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header

requirements/base.in

@@ -4,9 +4,11 @@ argon2-cffi
 whitenoise
 redis
 hiredis
+httpx[http2]
 celery
 jsonpath-ng
 xml2json @ git+https://github.com/dimagi/xml2json@041b1ef
+quickcache
 
 # Django
 # ------------------------------------------------------------------------------

requirements/base.txt

@@ -6,6 +6,8 @@
 #
 amqp==5.1.1
     # via kombu
+anyio==3.7.1
+    # via httpcore
 argon2-cffi==21.3.0
     # via -r requirements/base.in
 argon2-cffi-bindings==21.2.0
@@ -25,7 +27,10 @@ celery==5.3.1
     #   -r requirements/base.in
     #   django-celery-beat
 certifi==2023.7.22
-    # via requests
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
 cffi==1.15.1
     # via
     #   argon2-cffi-bindings
@@ -91,10 +96,27 @@ djangorestframework==3.14.0
     #   drf-spectacular
 drf-spectacular==0.26.4
     # via -r requirements/base.in
+exceptiongroup==1.1.2
+    # via anyio
+h11==0.14.0
+    # via httpcore
+h2==4.1.0
+    # via httpx
 hiredis==2.2.3
     # via -r requirements/base.in
+hpack==4.0.0
+    # via h2
+httpcore==0.17.3
+    # via httpx
+httpx[http2]==0.24.1
+    # via -r requirements/base.in
+hyperframe==6.0.1
+    # via h2
 idna==3.4
-    # via requests
+    # via
+    #   anyio
+    #   httpx
+    #   requests
 inflection==0.5.1
     # via drf-spectacular
 jsonpath-ng==1.5.3
@@ -135,6 +157,8 @@ pytz==2023.3
     #   djangorestframework
 pyyaml==6.0.1
     # via drf-spectacular
+quickcache==0.5.4
+    # via -r requirements/base.in
 redis==4.6.0
     # via
     #   -r requirements/base.in
@@ -157,6 +181,12 @@ six==1.16.0
     # via
     #   jsonpath-ng
     #   python-dateutil
+    #   quickcache
+sniffio==1.3.0
+    # via
+    #   anyio
+    #   httpcore
+    #   httpx
 sqlparse==0.4.4
     # via django
 text-unidecode==1.3

requirements/dev.txt

@@ -5,7 +5,9 @@
 #    inv requirements
 #
 anyio==3.7.1
-    # via watchfiles
+    # via
+    #   -c requirements/base.txt
+    #   watchfiles
 asgiref==3.7.2
     # via
     #   -c requirements/base.txt
@@ -48,6 +50,7 @@ django-debug-toolbar==4.1.0
     # via -r requirements/dev.in
 exceptiongroup==1.1.2
     # via
+    #   -c requirements/base.txt
     #   anyio
     #   pytest
 executing==1.2.0
@@ -173,7 +176,9 @@ six==1.16.0
     #   asttokens
     #   python-dateutil
 sniffio==1.3.0
-    # via anyio
+    # via
+    #   -c requirements/base.txt
+    #   anyio
 sqlparse==0.4.4
     # via
     #   -c requirements/base.txt