Fix release notes generation (modelica#3999)

* Fix release notes generation The previously used docverter service for MD to HTML/PDF conversion was shut down and will not come back. This is an alternative implementation with local MD to HTML conversion and utilizing the docraptor service for HTML to PDF conversion. * Bump dependencies * Bump python packages
henrikt-ma · Jan 19, 2024 · a47caa4 · a47caa4
1 parent 8753b87
commit a47caa4
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 88 deletions.
diff --git a/.github/workflows/generateReleaseNotes.yml b/.github/workflows/generateReleaseNotes.yml
@@ -11,6 +11,11 @@ on:
         description: 'Version of the release'
         default: '4.1.0'
         required: true
+      watermark:
+        description: 'PDF contains watermark for test'
+        default: true
+        required: true
+        type: boolean
 
 jobs:
   generate_release_notes:
@@ -25,9 +30,12 @@ jobs:
         with:
           python-version: 3.8
       - name: Install python packages
-        run: pip install --user requests
+        run: pip install --disable-pip-version-check --user docraptor==3.1.0 Jinja2==3.1.3 Markdown==3.5.2 requests PyYAML==6.0.1
       - name: Generate MSL release notes from closed GitHub issues
         timeout-minutes: 3
+        env:
+          DOCRAPTOR_API_KEY: ${{ secrets.DOCRAPTOR_API_KEY }}
+          DOCRAPTOR_TEST: ${{ github.event.inputs.watermark }}
         run: python ./Modelica/Resources/Documentation/Generate-ReleaseNotes.py ${{ github.event.inputs.milestone }} ${{ github.event.inputs.version }}
       - uses: actions/upload-artifact@v3
         with:

diff --git a/.gitignore b/.gitignore
@@ -29,7 +29,6 @@ buildlog.txt
 config.*
 configure
 depcomp
-docverter.css
 dsfinal.txt
 dsin.txt
 dslog.txt

diff --git a/Modelica/Resources/Documentation/Generate-ReleaseNotes.py b/Modelica/Resources/Documentation/Generate-ReleaseNotes.py
@@ -9,13 +9,18 @@
 python Generate-ReleaseNotes.py [milestone_number] [version_string]
 '''
 
-import requests
+from collections import defaultdict
+from enum import IntEnum
 import json
-import re
 import os
+import re
 import sys
-from enum import IntEnum
-from collections import defaultdict
+
+import docraptor
+import jinja2
+import markdown
+import requests
+import yaml
 
 IssueType = IntEnum(
     value='IssueType',
@@ -31,7 +36,7 @@
 def main(dir, milestone, version, auth):
     owner = 'modelica'
     repo = 'ModelicaStandardLibrary'
-    template = 'Modelica-ReleaseNotes-Template.html'
+    html_template = 'Modelica-ReleaseNotes-Template.html'
 
     # Filter closed issues (including pull requests) on milestone in ascending order
     p = {'state': 'closed', 'milestone': milestone, 'page': 1, 'per_page': 100, 'direction': 'asc'}
@@ -110,36 +115,35 @@ def main(dir, milestone, version, auth):
                     f.write("* [\#{1}]({2}) {0}\n".format(t, n, url))
                 f.write('\n')
 
+    # Read again Markdown
     with open(os.path.join(path, 'ResolvedGitHubIssues.md'), 'r') as f:
-        content = f.read()
+        md = f.read()
 
     # Convert Markdown -> HTML
-    with open(os.path.join(path, 'ResolvedGitHubIssues.html'), 'wb') as f:
-        url = 'http://c.docverter.com/convert'
-        title = 'MSL {0} GitHub issues'.format(version)
-        data = {'to': 'html', 'from': 'markdown', 'template': template, 'title': title}
-        files = { \
-            'input_files[]': open(os.path.join(path, 'ResolvedGitHubIssues.md'), 'rb'), \
-            'other_files[]': open(os.path.join(dir, template), 'rb') \
-        }
-        r = requests.post(url, data=data, files=files)
-        f.write(r.content)
+    with open(os.path.join(dir, html_template), 'r') as f:
+        template = f.read()
+    html = jinja2.Template(template).render(
+        content=markdown.markdown(md),
+        title='MSL {0} GitHub issues'.format(version))
+    with open(os.path.join(path, 'ResolvedGitHubIssues.html'), 'w') as f:
+        f.write(html)
 
     # Convert Markdown -> PDF
-    with open(os.path.join(path, 'ResolvedGitHubIssues.pdf'), 'wb') as f:
-        url = 'http://c.docverter.com/convert'
-        css = 'docverter.css'
-        with open(os.path.join(dir, css), 'w') as c:
-            pageInfo = '@page {size: A4 portrait;}'
-            c.write(pageInfo)
-        data = {'to': 'pdf', 'from': 'markdown', 'css': css, 'template': template}
-        files = [ \
-            ('input_files[]', ('ResolvedGitHubIssues.md', open(os.path.join(path, 'ResolvedGitHubIssues.md'), 'rb'), 'text/markdown')), \
-            ('other_files[]', (template, open(os.path.join(dir, template), 'rb'), 'text/html')), \
-            ('other_files[]', (css, open(os.path.join(dir, css), 'rb'), 'text/css')) \
-        ]
-        r = requests.post(url, data=data, files=files)
-        f.write(r.content)
+    doc_api = docraptor.DocApi()
+    doc_api.api_client.configuration.username = os.getenv('DOCRAPTOR_API_KEY', default='YOUR_API_KEY_HERE')
+    try:
+        pdf = doc_api.create_doc({
+            'test': yaml.load(os.getenv('DOCRAPTOR_TEST', default='True'), yaml.SafeLoader),
+            'name': 'ResolvedGitHubIssues.pdf',
+            'document_content': html,
+            'document_type': 'pdf'})
+        with open(os.path.join(path, 'ResolvedGitHubIssues.pdf'), 'wb') as f:
+            f.write(pdf)
+    except docraptor.rest.ApiException as e:
+        print(e.status)
+        print(e.reason)
+        print(e.body)
+
 
 if __name__ == '__main__':
     module_dir, module_name = os.path.split(__file__)

diff --git a/Modelica/Resources/Documentation/Modelica-ReleaseNotes-Template.html b/Modelica/Resources/Documentation/Modelica-ReleaseNotes-Template.html
@@ -1,71 +1,19 @@
-<!-- See https://github.com/jgm/pandoc-templates -->
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"$if(lang)$ lang="$lang$" xml:lang="$lang$"$endif$$if(dir)$ dir="$dir$"$endif$>
+<html xmlns="http://www.w3.org/1999/xhtml">
 <head>
   <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
   <meta http-equiv="Content-Style-Type" content="text/css" />
-  <meta name="generator" content="pandoc" />
-$for(author-meta)$
-  <meta name="author" content="$author-meta$" />
-$endfor$
-$if(date-meta)$
-  <meta name="date" content="$date-meta$" />
-$endif$
-$if(keywords)$
-  <meta name="keywords" content="$for(keywords)$$keywords$$sep$, $endfor$" />
-$endif$
-  <title>$if(title-prefix)$$title-prefix$$endif$</title>
+  <title>{{ title }}</title>
   <style type="text/css">
       h1, h2, h3, li, p{font-family: sans-serif;}
       code{white-space: pre-wrap;}
       span.smallcaps{font-variant: small-caps;}
       span.underline{text-decoration: underline;}
       div.column{display: inline-block; vertical-align: top; width: 50%;}
-$if(quotes)$
-      q { quotes: "“" "”" "‘" "’"; }
-$endif$
+      @page {size: A4 portrait;}
   </style>
-$if(highlighting-css)$
-  <style type="text/css">
-$highlighting-css$
-  </style>
-$endif$
-$for(css)$
-  <link rel="stylesheet" href="$css$" type="text/css" />
-$endfor$
-$if(math)$
-  $math$
-$endif$
-$for(header-includes)$
-  $header-includes$
-$endfor$
 </head>
 <body>
-$for(include-before)$
-$include-before$
-$endfor$
-$if(title)$
-<div id="$idprefix$header">
-<h1 class="title">$title$</h1>
-$if(subtitle)$
-<h1 class="subtitle">$subtitle$</h1>
-$endif$
-$for(author)$
-<h2 class="author">$author$</h2>
-$endfor$
-$if(date)$
-<h3 class="date">$date$</h3>
-$endif$
-</div>
-$endif$
-$if(toc)$
-<div id="$idprefix$TOC">
-$table-of-contents$
-</div>
-$endif$
-$body$
-$for(include-after)$
-$include-after$
-$endfor$
+{{ content }}
 </body>
 </html>