api: add maintenance module

mpcontribs-api/Dockerfile

@@ -50,6 +50,7 @@ EXPOSE 5000 5002 5003 5005
 COPY start.sh .
 COPY main.py .
 COPY supervisord.conf .
+COPY maintenance.py .
 RUN chmod +x main.py start.sh
 
 CMD ["/usr/bin/supervisord", "-c", "supervisord.conf"]

mpcontribs-api/maintenance.py

@@ -0,0 +1,47 @@
+# make sure correct units are indicated in project.columns before running this
+
+from boltons.iterutils import remap
+from mongoengine.queryset.visitor import Q
+
+from mpcontribs.api import enter
+from mpcontribs.api.projects.document import Projects
+from mpcontribs.api.contributions.document import Contributions
+
+
+def visit(path, key, value):
+    if isinstance(value, dict) and "display" in value:
+        return key, value["display"]
+    return True
+
+
+def fix_units(name):
+    fields = list(Contributions._fields.keys())
+    project = Projects.objects.with_id(name).reload("columns")
+    query = Q()
+
+    for column in project.columns:
+        if column.unit and column.unit != "NaN":
+            path = column.path.replace(".", "__")
+            q = {f"{path}__unit__ne": column["unit"]}
+            query |= Q(**q)
+
+    contribs = Contributions.objects(Q(project=name) & query).only(*fields)
+    num = contribs.count()
+    print(name, num)
+
+    for idx, contrib in enumerate(contribs):
+        contrib.data = remap(contrib.data, visit=visit, enter=enter)  # pull out display
+        contrib.save(signal_kwargs={"skip": True})  # reparse display with intended unit
+
+        if idx and not idx%100:
+            print(idx)
+
+    if num:
+        print("post_save ...")
+        Contributions.post_save(Contributions, contrib)
+
+    print("DONE")
+
+# additional maintenance functions
+# TODO clean notebooks
+# TODO update_projects/stats