Switch build scripts from PyYAML to ruamel.yaml (re #275)

bin/create_hugo_pages.py

@@ -32,19 +32,27 @@
 from glob import glob
 from slugify import slugify
 from tqdm import tqdm
+import io
 import logging as log
 import os
 import shutil
-import yaml
+import ruamel.yaml
 
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader
+yaml = ruamel.yaml.YAML()
+yaml.version = "1.1"
+yaml.default_flow_style = False
 
 from anthology.utils import SeverityTracker
 
 
+def yaml_dump(data, stream=None):
+    tmp = io.StringIO()
+    yaml.dump(data, stream=tmp)
+    stream.write("---\n")
+    stream.write(tmp.getvalue().replace("%YAML 1.1\n---\n", ""))
+    stream.write("---\n")
+
+
 def check_directory(cdir, clean=False):
     if not os.path.isdir(cdir) and not os.path.exists(cdir):
         os.mkdir(cdir)
@@ -77,7 +85,7 @@ def create_papers(srcdir, clean=False):
     for yamlfile in tqdm(glob("{}/data/papers/*.yaml".format(srcdir))):
         log.debug("Processing {}".format(yamlfile))
         with open(yamlfile, "r") as f:
-            data = yaml.load(f, Loader=Loader)
+            data = yaml.load(f)
         # Create a paper stub for each entry in the volume
         for anthology_id, entry in data.items():
             paper_dir = "{}/content/papers/{}/{}".format(
@@ -86,13 +94,9 @@ def create_papers(srcdir, clean=False):
             if not os.path.exists(paper_dir):
                 os.makedirs(paper_dir)
             with open("{}/{}.md".format(paper_dir, anthology_id), "w") as f:
-                print("---", file=f)
-                yaml.dump(
-                    {"anthology_id": anthology_id, "title": entry["title"]},
-                    default_flow_style=False,
-                    stream=f,
+                yaml_dump(
+                    {"anthology_id": anthology_id, "title": entry["title"]}, stream=f
                 )
-                print("---", file=f)
 
 
 def create_volumes(srcdir, clean=False):
@@ -104,21 +108,18 @@ def create_volumes(srcdir, clean=False):
     yamlfile = "{}/data/volumes.yaml".format(srcdir)
     log.debug("Processing {}".format(yamlfile))
     with open(yamlfile, "r") as f:
-        data = yaml.load(f, Loader=Loader)
+        data = yaml.load(f)
     # Create a paper stub for each proceedings volume
     for anthology_id, entry in data.items():
         with open("{}/content/volumes/{}.md".format(srcdir, anthology_id), "w") as f:
-            print("---", file=f)
-            yaml.dump(
+            yaml_dump(
                 {
                     "anthology_id": anthology_id,
                     "title": entry["title"],
                     "slug": slugify(entry["title"]),
                 },
-                default_flow_style=False,
                 stream=f,
             )
-            print("---", file=f)
 
     return data
 
@@ -132,7 +133,7 @@ def create_people(srcdir, clean=False):
     for yamlfile in tqdm(glob("{}/data/people/*.yaml".format(srcdir))):
         log.debug("Processing {}".format(yamlfile))
         with open(yamlfile, "r") as f:
-            data = yaml.load(f, Loader=Loader)
+            data = yaml.load(f)
         # Create a page stub for each person
         for name, entry in data.items():
             # Only create page stub when name doesn't link to a canonical entry
@@ -147,10 +148,8 @@ def create_people(srcdir, clean=False):
                 "lastname": entry["last"],
             }
             with open("{}/{}.md".format(person_dir, name), "w") as f:
-                print("---", file=f)
                 # "lastname" is dumped to allow sorting by it in Hugo
-                yaml.dump(yaml_data, default_flow_style=False, stream=f)
-                print("---", file=f)
+                yaml_dump(yaml_data, stream=f)
 
     return data
 
@@ -160,7 +159,7 @@ def create_venues_and_events(srcdir, clean=False):
     yamlfile = "{}/data/venues.yaml".format(srcdir)
     log.debug("Processing {}".format(yamlfile))
     with open(yamlfile, "r") as f:
-        data = yaml.load(f, Loader=Loader)
+        data = yaml.load(f)
 
     log.info("Creating stubs for venues...")
     if not check_directory("{}/content/venues".format(srcdir), clean=clean):
@@ -169,13 +168,11 @@ def create_venues_and_events(srcdir, clean=False):
     for venue, venue_data in data.items():
         venue_str = venue_data["slug"]
         with open("{}/content/venues/{}.md".format(srcdir, venue_str), "w") as f:
-            print("---", file=f)
             yaml_data = {"venue": venue, "title": venue_data["name"]}
             if venue_data["is_toplevel"]:
                 main_letter = venue_data["main_letter"]
                 yaml_data["aliases"] = ["/papers/{}/".format(main_letter)]
-            yaml.dump(yaml_data, default_flow_style=False, stream=f)
-            print("---", file=f)
+            yaml_dump(yaml_data, stream=f)
 
     log.info("Creating stubs for events...")
     if not check_directory("{}/content/events".format(srcdir), clean=clean):
@@ -187,7 +184,6 @@ def create_venues_and_events(srcdir, clean=False):
             with open(
                 "{}/content/events/{}-{}.md".format(srcdir, venue_str, year), "w"
             ) as f:
-                print("---", file=f)
                 yaml_data = {
                     "venue": venue,
                     "year": year,
@@ -199,16 +195,15 @@ def create_venues_and_events(srcdir, clean=False):
                     yaml_data["aliases"] = [
                         "/papers/{}/{}/".format(main_letter, main_prefix)
                     ]
-                yaml.dump(yaml_data, default_flow_style=False, stream=f)
-                print("---", file=f)
+                yaml_dump(yaml_data, stream=f)
 
 
 def create_sigs(srcdir, clean=False):
     """Creates page stubs for all SIGs in the Anthology."""
     yamlfile = "{}/data/sigs.yaml".format(srcdir)
     log.debug("Processing {}".format(yamlfile))
     with open(yamlfile, "r") as f:
-        data = yaml.load(f, Loader=Loader)
+        data = yaml.load(f)
 
     log.info("Creating stubs for SIGs...")
     if not check_directory("{}/content/sigs".format(srcdir), clean=clean):
@@ -217,17 +212,14 @@ def create_sigs(srcdir, clean=False):
     for sig, sig_data in data.items():
         sig_str = sig_data["slug"]
         with open("{}/content/sigs/{}.md".format(srcdir, sig_str), "w") as f:
-            print("---", file=f)
-            yaml.dump(
+            yaml_dump(
                 {
                     "acronym": sig,
                     "short_acronym": sig[3:] if sig.startswith("SIG") else sig,
                     "title": sig_data["name"],
                 },
-                default_flow_style=False,
                 stream=f,
             )
-            print("---", file=f)
 
 
 if __name__ == "__main__":

bin/create_hugo_yaml.py

@@ -32,12 +32,10 @@
 from tqdm import tqdm
 import logging as log
 import os
-import yaml
+import ruamel.yaml
 
-try:
-    from yaml import CSafeDumper as Dumper
-except ImportError:
-    from yaml import SafeDumper as Dumper
+yaml = ruamel.yaml.YAML()
+yaml.version = "1.1"
 
 from anthology import Anthology
 from anthology.utils import SeverityTracker
@@ -166,24 +164,24 @@ def export_anthology(anthology, outdir, dryrun=False):
         progress = tqdm(total=len(papers) + len(people) + 7)
         for top_level_id, paper_list in papers.items():
             with open("{}/papers/{}.yaml".format(outdir, top_level_id), "w") as f:
-                yaml.dump(paper_list, Dumper=Dumper, stream=f)
+                yaml.dump(paper_list, stream=f)
             progress.update()
 
         with open("{}/volumes.yaml".format(outdir), "w") as f:
-            yaml.dump(volumes, Dumper=Dumper, stream=f)
+            yaml.dump(volumes, stream=f)
         progress.update(5)
 
         with open("{}/venues.yaml".format(outdir), "w") as f:
-            yaml.dump(venues, Dumper=Dumper, stream=f)
+            yaml.dump(venues, stream=f)
         progress.update()
 
         with open("{}/sigs.yaml".format(outdir), "w") as f:
-            yaml.dump(sigs, Dumper=Dumper, stream=f)
+            yaml.dump(sigs, stream=f)
         progress.update()
 
         for first_letter, people_list in people.items():
             with open("{}/people/{}.yaml".format(outdir, first_letter), "w") as f:
-                yaml.dump(people_list, Dumper=Dumper, stream=f)
+                yaml.dump(people_list, stream=f)
             progress.update()
         progress.close()
 

bin/requirements.txt

@@ -3,6 +3,7 @@ latexcodec
 lxml>=4.2.0
 python-slugify>=2.0
 PyYAML>=3.0
+ruamel.yaml==0.15.93
 stop-words
 tqdm
 texsoup<=0.1.4