feat: Import IESG artifacts into the datatracker (#6908)

* chore: remove unused setting

* feat: initial import of iesg minutes

* fix: let the meetings view show older iesg meetings

* feat: iesg narrative minutes

* feat: import bof coordination call minutes

* wip: import commands for iesg appeals and statements

* feat: import iesg statements.

* feat: import iesg artifacts

* feat: many fewer n+1 queries for the group meetings view

* fix: restore chain of elifs in views_doc

* fix: use self.stdout.write vs print in mgmt commands

* fix: use replace instead of astimezone when appropriate

* chore: refactor new migrations into one

* fix: transcode some old files into utf8

* fix: repair overzealous replace

* chore: black

* fix: address minro review comments

* fix: actually capture transcoding work

* fix: handle multiple iesg statements on the same day

* fix: better titles

* feat: pill badge replaced statements

* fix: consolodate source repos to one

* feat: liberal markdown for secretariat controlled content

* fix: handle (and clean) html narrative minutes

* feat: scrub harder

* fix: simplify and improve a scrubber

* chore: reorder migrations

ietf/api/tests.py

@@ -368,7 +368,7 @@ def test_api_upload_polls_and_chatlog(self):
             r = self.client.post(url,{'apikey':apikey.hash(),'apidata': f'{{"session_id":{session.pk}, "{type_id}":{content}}}'})
             self.assertEqual(r.status_code, 200)
 
-            newdoc = session.sessionpresentation_set.get(document__type_id=type_id).document
+            newdoc = session.presentations.get(document__type_id=type_id).document
             newdoccontent = get_unicode_document_content(newdoc.name, Path(session.meeting.get_materials_path()) / type_id / newdoc.uploaded_filename)
             self.assertEqual(json.loads(content), json.loads(newdoccontent))
 
@@ -454,7 +454,7 @@ def test_deprecated_api_upload_bluesheet(self):
                                    'item': '1', 'bluesheet': bluesheet, })
         self.assertContains(r, "Done", status_code=200)
 
-        bluesheet = session.sessionpresentation_set.filter(document__type__slug='bluesheets').first().document
+        bluesheet = session.presentations.filter(document__type__slug='bluesheets').first().document
         # We've submitted an update; check that the rev is right
         self.assertEqual(bluesheet.rev, '01')
         # Check the content
@@ -569,7 +569,7 @@ def test_api_upload_bluesheet(self):
         self.assertContains(r, "Done", status_code=200)
 
         bluesheet = (
-            session.sessionpresentation_set.filter(document__type__slug="bluesheets")
+            session.presentations.filter(document__type__slug="bluesheets")
             .first()
             .document
         )

ietf/doc/migrations/0021_narrativeminutes.py

@@ -0,0 +1,39 @@
+# Copyright The IETF Trust 2023, All Rights Reserved
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    StateType = apps.get_model("doc", "StateType")
+    State = apps.get_model("doc", "State")
+
+    StateType.objects.create(
+        slug="narrativeminutes",
+        label="State",
+    )
+    for order, slug in enumerate(["active", "deleted"]):
+        State.objects.create(
+            slug=slug,
+            type_id="narrativeminutes",
+            name=slug.capitalize(),
+            order=order,
+            desc="",
+            used=True,
+        )
+
+
+def reverse(apps, schema_editor):
+    StateType = apps.get_model("doc", "StateType")
+    State = apps.get_model("doc", "State")
+
+    State.objects.filter(type_id="narrativeminutes").delete()
+    StateType.objects.filter(slug="narrativeminutes").delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0020_move_errata_tags"),
+        ("name", "0013_narrativeminutes"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]

ietf/doc/models.py

@@ -148,7 +148,7 @@ def get_file_path(self):
                     else:
                         self._cached_file_path = settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR
             elif self.meeting_related() and self.type_id in (
-                    "agenda", "minutes", "slides", "bluesheets", "procmaterials", "chatlog", "polls"
+                    "agenda", "minutes", "narrativeminutes", "slides", "bluesheets", "procmaterials", "chatlog", "polls"
             ):
                 meeting = self.get_related_meeting()
                 if meeting is not None:
@@ -438,7 +438,7 @@ def has_rfc_editor_note(self):
         return e != None and (e.text != "")
 
     def meeting_related(self):
-        if self.type_id in ("agenda","minutes","bluesheets","slides","recording","procmaterials","chatlog","polls"):
+        if self.type_id in ("agenda","minutes", "narrativeminutes", "bluesheets","slides","recording","procmaterials","chatlog","polls"):
              return self.type_id != "slides" or self.get_state_slug('reuse_policy')=='single'
         return False
 
@@ -1028,7 +1028,7 @@ def related_ipr(self):
     def future_presentations(self):
         """ returns related SessionPresentation objects for meetings that
             have not yet ended. This implementation allows for 2 week meetings """
-        candidate_presentations = self.sessionpresentation_set.filter(
+        candidate_presentations = self.presentations.filter(
             session__meeting__date__gte=date_today() - datetime.timedelta(days=15)
         )
         return sorted(
@@ -1041,11 +1041,11 @@ def last_presented(self):
         """ returns related SessionPresentation objects for the most recent meeting in the past"""
         # Assumes no two meetings have the same start date - if the assumption is violated, one will be chosen arbitrarily
         today = date_today()
-        candidate_presentations = self.sessionpresentation_set.filter(session__meeting__date__lte=today)
+        candidate_presentations = self.presentations.filter(session__meeting__date__lte=today)
         candidate_meetings = set([p.session.meeting for p in candidate_presentations if p.session.meeting.end_date()<today])
         if candidate_meetings:
             mtg = sorted(list(candidate_meetings),key=lambda x:x.date,reverse=True)[0]
-            return self.sessionpresentation_set.filter(session__meeting=mtg)
+            return self.presentations.filter(session__meeting=mtg)
         else:
             return None
 

ietf/doc/tests.py

@@ -2529,8 +2529,8 @@ def setUp(self):
 
     def test_view_document_meetings(self):
         doc = IndividualDraftFactory.create()
-        doc.sessionpresentation_set.create(session=self.inprog,rev=None)
-        doc.sessionpresentation_set.create(session=self.interim,rev=None)
+        doc.presentations.create(session=self.inprog,rev=None)
+        doc.presentations.create(session=self.interim,rev=None)
 
         url = urlreverse('ietf.doc.views_doc.all_presentations', kwargs=dict(name=doc.name))
         response = self.client.get(url)
@@ -2541,8 +2541,8 @@ def test_view_document_meetings(self):
         self.assertFalse(q('#addsessionsbutton'))
         self.assertFalse(q("a.btn:contains('Remove document')"))
 
-        doc.sessionpresentation_set.create(session=self.past_cutoff,rev=None)
-        doc.sessionpresentation_set.create(session=self.past,rev=None)
+        doc.presentations.create(session=self.past_cutoff,rev=None)
+        doc.presentations.create(session=self.past,rev=None)
 
         self.client.login(username="secretary", password="secretary+password")
         response = self.client.get(url)
@@ -2577,7 +2577,7 @@ def test_view_document_meetings(self):
 
     def test_edit_document_session(self):
         doc = IndividualDraftFactory.create()
-        sp = doc.sessionpresentation_set.create(session=self.future,rev=None)
+        sp = doc.presentations.create(session=self.future,rev=None)
 
         url = urlreverse('ietf.doc.views_doc.edit_sessionpresentation',kwargs=dict(name='no-such-doc',session_id=sp.session_id))
         response = self.client.get(url)
@@ -2604,12 +2604,12 @@ def test_edit_document_session(self):
         self.assertEqual(1,doc.docevent_set.count())
         response = self.client.post(url,{'version':'00','save':''})
         self.assertEqual(response.status_code, 302)
-        self.assertEqual(doc.sessionpresentation_set.get(pk=sp.pk).rev,'00')
+        self.assertEqual(doc.presentations.get(pk=sp.pk).rev,'00')
         self.assertEqual(2,doc.docevent_set.count())
 
     def test_edit_document_session_after_proceedings_closed(self):
         doc = IndividualDraftFactory.create()
-        sp = doc.sessionpresentation_set.create(session=self.past_cutoff,rev=None)
+        sp = doc.presentations.create(session=self.past_cutoff,rev=None)
 
         url = urlreverse('ietf.doc.views_doc.edit_sessionpresentation',kwargs=dict(name=doc.name,session_id=sp.session_id))
         self.client.login(username=self.group_chair.user.username,password='%s+password'%self.group_chair.user.username)
@@ -2624,7 +2624,7 @@ def test_edit_document_session_after_proceedings_closed(self):
 
     def test_remove_document_session(self):
         doc = IndividualDraftFactory.create()
-        sp = doc.sessionpresentation_set.create(session=self.future,rev=None)
+        sp = doc.presentations.create(session=self.future,rev=None)
 
         url = urlreverse('ietf.doc.views_doc.remove_sessionpresentation',kwargs=dict(name='no-such-doc',session_id=sp.session_id))
         response = self.client.get(url)
@@ -2649,12 +2649,12 @@ def test_remove_document_session(self):
         self.assertEqual(1,doc.docevent_set.count())
         response = self.client.post(url,{'remove_session':''})
         self.assertEqual(response.status_code, 302)
-        self.assertFalse(doc.sessionpresentation_set.filter(pk=sp.pk).exists())
+        self.assertFalse(doc.presentations.filter(pk=sp.pk).exists())
         self.assertEqual(2,doc.docevent_set.count())
 
     def test_remove_document_session_after_proceedings_closed(self):
         doc = IndividualDraftFactory.create()
-        sp = doc.sessionpresentation_set.create(session=self.past_cutoff,rev=None)
+        sp = doc.presentations.create(session=self.past_cutoff,rev=None)
 
         url = urlreverse('ietf.doc.views_doc.remove_sessionpresentation',kwargs=dict(name=doc.name,session_id=sp.session_id))
         self.client.login(username=self.group_chair.user.username,password='%s+password'%self.group_chair.user.username)

ietf/doc/views_doc.py

@@ -832,7 +832,7 @@ def document_main(request, name, rev=None, document_html=False):
                                        sorted_relations=sorted_relations,
                                        ))
 
-    elif doc.type_id in ("slides", "agenda", "minutes", "bluesheets", "procmaterials",):
+    elif doc.type_id in ("slides", "agenda", "minutes", "narrativeminutes", "bluesheets", "procmaterials",):
         can_manage_material = can_manage_materials(request.user, doc.group)
         presentations = doc.future_presentations()
         if doc.uploaded_filename:
@@ -916,9 +916,9 @@ def document_main(request, name, rev=None, document_html=False):
 
     elif doc.type_id in ("chatlog", "polls"):
         if isinstance(doc,DocHistory):
-            session = doc.doc.sessionpresentation_set.last().session
+            session = doc.doc.presentations.last().session
         else:
-            session = doc.sessionpresentation_set.last().session
+            session = doc.presentations.last().session
         pathname = Path(session.meeting.get_materials_path()) / doc.type_id / doc.uploaded_filename
         content = get_unicode_document_content(doc.name, str(pathname))
         return render(
@@ -943,7 +943,7 @@ def document_main(request, name, rev=None, document_html=False):
         variants = set([match.name.split(".")[1] for match in Path(doc.get_file_path()).glob(f"{basename}.*")])
         inlineable = any([ext in variants for ext in ["md", "txt"]])
         if inlineable:
-            content = markdown.markdown(doc.text_or_error())
+            content = markdown.liberal_markdown(doc.text_or_error())
         else:
             content = "No format available to display inline"
             if "pdf" in variants:
@@ -2057,7 +2057,7 @@ def __init__(self, *args, **kwargs):
 
 def edit_sessionpresentation(request,name,session_id):
     doc = get_object_or_404(Document, name=name)
-    sp = get_object_or_404(doc.sessionpresentation_set, session_id=session_id)
+    sp = get_object_or_404(doc.presentations, session_id=session_id)
 
     if not sp.session.can_manage_materials(request.user):
         raise Http404
@@ -2074,7 +2074,7 @@ def edit_sessionpresentation(request,name,session_id):
         if form.is_valid():
             new_selection = form.cleaned_data['version']
             if initial['version'] != new_selection:
-                doc.sessionpresentation_set.filter(pk=sp.pk).update(rev=None if new_selection=='current' else new_selection)
+                doc.presentations.filter(pk=sp.pk).update(rev=None if new_selection=='current' else new_selection)
                 c = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=request.user.person)
                 c.desc = "Revision for session %s changed to  %s" % (sp.session,new_selection)
                 c.save()
@@ -2086,7 +2086,7 @@ def edit_sessionpresentation(request,name,session_id):
 
 def remove_sessionpresentation(request,name,session_id):
     doc = get_object_or_404(Document, name=name)
-    sp = get_object_or_404(doc.sessionpresentation_set, session_id=session_id)
+    sp = get_object_or_404(doc.presentations, session_id=session_id)
 
     if not sp.session.can_manage_materials(request.user):
         raise Http404
@@ -2095,7 +2095,7 @@ def remove_sessionpresentation(request,name,session_id):
         raise Http404
 
     if request.method == 'POST':
-        doc.sessionpresentation_set.filter(pk=sp.pk).delete()
+        doc.presentations.filter(pk=sp.pk).delete()
         c = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=request.user.person)
         c.desc = "Removed from session: %s" % (sp.session)
         c.save()
@@ -2119,7 +2119,7 @@ def add_sessionpresentation(request,name):
     version_choices.insert(0,('current','Current at the time of the session'))
 
     sessions = get_upcoming_manageable_sessions(request.user)
-    sessions = sort_sessions([s for s in sessions if not s.sessionpresentation_set.filter(document=doc).exists()])
+    sessions = sort_sessions([s for s in sessions if not s.presentations.filter(document=doc).exists()])
     if doc.group:
         sessions = sorted(sessions,key=lambda x:0 if x.group==doc.group else 1)
 
@@ -2132,7 +2132,7 @@ def add_sessionpresentation(request,name):
             session_id = session_form.cleaned_data['session']
             version = version_form.cleaned_data['version']
             rev = None if version=='current' else version
-            doc.sessionpresentation_set.create(session_id=session_id,rev=rev)
+            doc.presentations.create(session_id=session_id,rev=rev)
             c = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=request.user.person)
             c.desc = "%s to session: %s" % ('Added -%s'%rev if rev else 'Added', Session.objects.get(pk=session_id))
             c.save()

ietf/doc/views_material.py

@@ -113,6 +113,8 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
     valid_doctypes = ['procmaterials']
     if group is not None:
         valid_doctypes.extend(['minutes','agenda','bluesheets'])
+        if group.acronym=="iesg":
+            valid_doctypes.append("narrativeminutes")
         valid_doctypes.extend(group.features.material_types)
 
     if document_type.slug not in valid_doctypes:

ietf/doc/views_statement.py

@@ -94,7 +94,7 @@ def require_field(f):
                     )
         if markdown_content != "":
             try:
-                _ = markdown.markdown(markdown_content)
+                _ = markdown.liberal_markdown(markdown_content)
             except Exception as e:
                 raise forms.ValidationError(f"Markdown processing failed: {e}")