Merge staging into production, 2023 Nov 10 edition

django/cantusdb_project/main_app/admin.py

@@ -79,6 +79,7 @@ def get_source_siglum(self, obj):
         "volpiano_notes",
         "volpiano_intervals",
         "title",
+        "differentiae_database",
     )
     form = AdminChantForm
     raw_id_fields = (
@@ -88,6 +89,13 @@ def get_source_siglum(self, obj):
     ordering = ("source__siglum",)
 
 
+class DifferentiaAdmin(BaseModelAdmin):
+    search_fields = (
+        "differentia_id",
+        "id",
+    )
+
+
 class FeastAdmin(BaseModelAdmin):
     search_fields = (
         "name",
@@ -164,6 +172,8 @@ def get_source_siglum(self, obj):
 
 
 class SourceAdmin(BaseModelAdmin):
+    exclude = ("source_status",)
+
     # These search fields are also available on the user-source inline relationship in the user admin page
     search_fields = (
         "siglum",
@@ -211,6 +221,7 @@ class SourceAdmin(BaseModelAdmin):
 
 admin.site.register(Century, CenturyAdmin)
 admin.site.register(Chant, ChantAdmin)
+admin.site.register(Differentia, DifferentiaAdmin)
 admin.site.register(Feast, FeastAdmin)
 admin.site.register(Genre, GenreAdmin)
 admin.site.register(Notation, NotationAdmin)

django/cantusdb_project/main_app/forms.py

@@ -12,6 +12,7 @@
     Provenance,
     Century,
     Sequence,
+    Differentia,
 )
 from .widgets import (
     TextInputWidget,
@@ -82,7 +83,7 @@ class Meta:
             "feast",
             "mode",
             "differentia",
-            "differentia_new",
+            "diff_db",
             "finalis",
             "extra",
             "chant_range",
@@ -103,14 +104,14 @@ class Meta:
             "marginalia": TextInputWidget(),
             # folio: defined below (required)
             # c_sequence: defined below (required)
-            # office: defined below (foreignkey)
-            # genre: defined below (foreignkey)
+            "office": autocomplete.ModelSelect2(url="office-autocomplete"),
+            "genre": autocomplete.ModelSelect2(url="genre-autocomplete"),
             "position": TextInputWidget(),
             "cantus_id": TextInputWidget(),
             "feast": autocomplete.ModelSelect2(url="feast-autocomplete"),
             "mode": TextInputWidget(),
             "differentia": TextInputWidget(),
-            "differentia_new": TextInputWidget(),
+            "diff_db": autocomplete.ModelSelect2(url="differentia-autocomplete"),
             "finalis": TextInputWidget(),
             "extra": TextInputWidget(),
             "chant_range": VolpianoInputWidget(),
@@ -136,22 +137,6 @@ class Meta:
         help_text="Each folio starts with '1'.",
     )
 
-    # We use NameModelChoiceField here so the dropdown list of office/mass displays the name
-    # instead of [name] + description
-    office = NameModelChoiceField(
-        queryset=Office.objects.all().order_by("name"),
-        required=False,
-    )
-    office.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
-
-    # We use NameModelChoiceField here so the dropdown list of genres displays the name
-    # instead of [name] + description
-    genre = NameModelChoiceField(
-        queryset=Genre.objects.all().order_by("name"),
-        required=False,
-    )
-    genre.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
-
     manuscript_full_text_std_spelling = forms.CharField(
         required=True,
         widget=TextAreaWidget,
@@ -206,6 +191,7 @@ class Meta:
         widgets = {
             "title": TextInputWidget(),
             "siglum": TextInputWidget(),
+            "provenance": autocomplete.ModelSelect2(url="provenance-autocomplete"),
             "provenance_notes": TextInputWidget(),
             "date": TextInputWidget(),
             "cursus": SelectWidget(),
@@ -238,13 +224,6 @@ class Meta:
             ),
         }
 
-    provenance = forms.ModelChoiceField(
-        queryset=Provenance.objects.all().order_by("name"), required=False
-    )
-    provenance.widget.attrs.update(
-        {"class": "form-control custom-select custom-select-sm"}
-    )
-
     TRUE_FALSE_CHOICES_SOURCE = (
         (True, "Full source"),
         (False, "Fragment or Fragmented"),
@@ -283,7 +262,7 @@ class Meta:
             "mode",
             "finalis",
             "differentia",
-            "differentia_new",
+            "diff_db",
             "extra",
             "image_link",
             "indexing_notes",
@@ -302,15 +281,15 @@ class Meta:
             # folio: defined below (required)
             # c_sequence: defined below (required)
             "feast": autocomplete.ModelSelect2(url="feast-autocomplete"),
-            # office: defined below (foreignkey)
-            # genre: defined below (foreignkey)
+            "office": autocomplete.ModelSelect2(url="office-autocomplete"),
+            "genre": autocomplete.ModelSelect2(url="genre-autocomplete"),
             "position": TextInputWidget(),
             "cantus_id": TextInputWidget(),
             "melody_id": TextInputWidget(),
             "mode": TextInputWidget(),
             "finalis": TextInputWidget(),
             "differentia": TextInputWidget(),
-            "differentia_new": TextInputWidget(),
+            "diff_db": autocomplete.ModelSelect2(url="differentia-autocomplete"),
             "extra": TextInputWidget(),
             "image_link": TextInputWidget(),
             "indexing_notes": TextAreaWidget(),
@@ -346,22 +325,6 @@ class Meta:
         help_text="Each folio starts with '1'.",
     )
 
-    # We use NameModelChoiceField here so the dropdown list of office/mass displays the name
-    # instead of [name] + description
-    office = NameModelChoiceField(
-        queryset=Office.objects.all().order_by("name"),
-        required=False,
-    )
-    office.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
-
-    # We use NameModelChoiceField here so the dropdown list of genres displays the name
-    # instead of [name] + description
-    genre = NameModelChoiceField(
-        queryset=Genre.objects.all().order_by("name"),
-        required=False,
-    )
-    genre.widget.attrs.update({"class": "form-control custom-select custom-select-sm"})
-
 
 class SourceEditForm(forms.ModelForm):
     class Meta:
@@ -396,6 +359,7 @@ class Meta:
             "title": TextInputWidget(),
             "rism_siglum": autocomplete.ModelSelect2(url="rismsiglum-autocomplete"),
             "siglum": TextInputWidget(),
+            "provenance": autocomplete.ModelSelect2(url="provenance-autocomplete"),
             "provenance_notes": TextInputWidget(),
             "date": TextInputWidget(),
             "summary": TextAreaWidget(),
@@ -427,13 +391,6 @@ class Meta:
             ),
         }
 
-    provenance = forms.ModelChoiceField(
-        queryset=Provenance.objects.all().order_by("name"), required=False
-    )
-    provenance.widget.attrs.update(
-        {"class": "form-control custom-select custom-select-sm"}
-    )  # adds styling
-
     CHOICES_FULL_SOURCE = (
         (None, "None"),
         (True, "Full source"),

django/cantusdb_project/main_app/management/commands/add_differentiae.py

@@ -0,0 +1,95 @@
+from main_app.models import Chant, Differentia
+from django.core.management.base import BaseCommand
+from typing import List, Optional
+from django.db.models.query import QuerySet
+
+
+# Python script to sync the differentia database with CantusDB. Previously, we had a CharField that represented
+# the differentia ID. Now, we change this field to a ForeignKey that points to the new Differentia model that contains
+# the ID, melodic transcription, and mode.
+
+
+DIFFERENTIA_DATA = "differentia_data.txt"
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        differentia_database: List[List[str]] = get_differentia_data(DIFFERENTIA_DATA)
+        total: int = len(differentia_database)
+        count: int = 0
+        for item in differentia_database:
+            differentia_id: str
+            melodic_transcription: str
+            mode: str
+            differentia_id, melodic_transcription, mode, _ = item
+            differentia: Optional[Differentia] = None
+            try:
+                differentia = Differentia.objects.get(differentia_id=differentia_id)
+            except Differentia.DoesNotExist:
+                pass
+
+            if differentia:
+                differentia.melodic_transcription = melodic_transcription
+                differentia.mode = mode
+                differentia.save()
+            else:
+                differentia = Differentia(
+                    differentia_id=differentia_id,
+                    melodic_transcription=melodic_transcription,
+                    mode=mode,
+                )
+                differentia.save()
+            if count % 100 == 0:
+                print(
+                    f"------------------- {count} of {total} differentia created -------------------"
+                )
+            count += 1
+
+        CHUNK_SIZE = 1_000
+        chants: QuerySet[Chant] = Chant.objects.all()
+        chants_count: int = chants.count()
+        start_index: int = 0
+        count: int = 0
+        while start_index <= chants_count:
+            print("processing chunk with start_index of", start_index)
+            chunk: QuerySet[Chant] = chants[start_index : start_index + CHUNK_SIZE]
+
+            # Update Chant model to use foreign key
+            for chant in chunk:
+                try:
+                    differentia_id: Optional[str] = chant.differentiae_database
+                    if not differentia_id is None:
+                        differentia = Differentia.objects.filter(
+                            differentia_id=differentia_id
+                        ).first()
+
+                        if differentia:
+                            chant.diff_db = differentia
+                        else:
+                            # If the Differentia doesn't exist, create a new one
+                            differentia: Differentia = Differentia(
+                                differentia_id=differentia_id,
+                            )
+                            differentia.save()
+                            chant.diff_db = differentia
+
+                        chant.save()
+                    if count % 100 == 0:
+                        print(
+                            f"------------------- {count} of {chants_count} chants updated -------------------"
+                        )
+                except Differentia.DoesNotExist:
+                    print(f"Differentia not found for chant: {chant}")
+                count += 1
+            del chunk  # make sure we don't use too much RAM
+            start_index += CHUNK_SIZE
+
+
+def get_differentia_data(filename) -> List[List[str]]:
+    differentia_list: List = []
+    with open(filename, "r") as file:
+        for line in file:
+            line: str = line.strip()
+            values: tuple = tuple(line.split(","))
+            differentia_list.append(values)
+    return differentia_list