portal: add maintenance generate_downloads

mpcontribs-portal/Dockerfile

@@ -70,5 +70,6 @@ COPY settings.py .
 COPY docker-entrypoint.sh .
 COPY manage.py .
 COPY start.sh .
+COPY maintenance.py .
 ENTRYPOINT ["/app/docker-entrypoint.sh"]
 CMD wait-for-it.sh $MPCONTRIBS_API_HOST -s -t 60 -- ./start.sh

mpcontribs-portal/maintenance.py

@@ -0,0 +1,33 @@
+import os
+import json
+
+from itertools import combinations
+from mpcontribs.portal.views import make_download
+from mpcontribs.client import Client
+
+FORMATS = ["json", "csv"]
+HEADERS = {"X-Authenticated-Groups": os.environ["ADMIN_GROUP"]}
+
+
+def generate_downloads():
+    client = Client(host=os.environ["MPCONTRIBS_API_HOST"], headers=HEADERS)
+    projects = client.projects.get_entries(
+        _fields=["name", "stats"]
+    ).result().get("data", [])
+    skip = {"columns", "contributions"}
+    print("PROJECTS", len(projects))
+
+    for project in projects:
+        name = project["name"]
+        include = [k for k, v in project["stats"].items() if k not in skip and v]
+
+        for fmt in FORMATS:
+            query = {"project": name, "format": fmt}
+            resp = make_download(client, query, [])
+            print(name, json.loads(resp.content))
+
+            if include:
+                for r in range(1, len(include)+1):
+                    for combo in combinations(include, r):
+                        resp = make_download(client, query, combo)
+                        print(name, combo, json.loads(resp.content))

mpcontribs-portal/mpcontribs/portal/views.py

@@ -469,16 +469,20 @@ def create_download(request):
         return make_download(client, query, include, timeout=20)
 
 
-def make_download(client, query, include, timeout=-1):
+def make_download(client, query, include=None, timeout=-1):
+    include = include or []
     key = _get_download_key(query, include)
     total_count, total_pages = client.get_totals(query=query, op="download")
 
     if total_count < 1:
         return JsonResponse({"error": "No results for query."})
 
+    kwargs = {
+        k: v for k, v in query.items()
+        if k not in {"format", "_sort", "_fields", "_limit", "per_page"}
+    }
     last_modified = client.contributions.get_entries(
-        _sort="-last_modified", _fields=["last_modified"], _limit=1,
-        **{k: v for k, v in query.items() if k not in {"format", "_sort"}}
+        _sort="-last_modified", _fields=["last_modified"], _limit=1, **kwargs
     ).result()["data"][0]["last_modified"]
 
     try: