Merge pull request #219 from byexamples/Document-Matrix-Version-Tested

Change some prints to make them machine-parseable

.gitignore

@@ -14,3 +14,4 @@ test/ds/django.conf
 test/ds/args
 prof-traces
 .release-notes
+.workflow-log

Makefile

@@ -257,13 +257,18 @@ release:
 	@X=`git describe --exact-match HEAD` && ( git tag -n1000 "$$X" | tail -n +3 | sed 's/^[[:blank:]]\{,4\}\(.*\)$$/\1/' | tee .release-notes | gh release create --generate-notes "$$X" --notes-files - )
 	@cat .release-notes
 
+runner-version-matrix:
+	gh auth status
+	@X=`gh run list --repo byexamples/byexample --workflow test --limit 1 | awk '{print $$(NF-2)}'` && gh run view --repo byexamples/byexample --log "$$X" > .workflow-log
+	@python test/runner_version_matrix.py .workflow-log
+
 clean_test:
 	@rm -f r.py
 	@rm -Rf w/
 	@mkdir -p w/
 
 clean: clean_test
-	rm -f .coverage .coverage.work.*
+	rm -f .coverage .coverage.work.* .release-notes .workflow-log
 	rm -Rf dist/ build/ *.egg-info
 	rm -Rf build/ *.egg-info
 	find . -name "*.pyc" -delete

byexample/modules/iasm.py

@@ -125,6 +125,9 @@ def __init__(self, verbosity, encoding, **unused):
 
         PexpectMixin.__init__(self, PS1_re=r':>', any_PS_re=r'[:-]>')
 
+    def __repr__(self):
+        return '%s Runner' % self.language
+
     def get_default_cmd(self, arch, mode, sz, pc, *args, **kargs):
         return "%e %p %a", {
             'e':

byexample/modules/powershell.py

@@ -131,6 +131,9 @@ def __init__(self, verbosity, encoding, **unused):
             self, PS1_re=r'byexample-ps1>', any_PS_re=r'(byexample-ps1>)|(>>)'
         )
 
+    def __repr__(self):
+        return 'PowerShell Runner'
+
     def get_default_cmd(self, *args, **kargs):
         return "%e %p %a", {
             'e':

byexample/runner.py

@@ -291,7 +291,8 @@ def _spawn_interpreter(
         if clog().isEnabledFor(INFO):
             v = self.get_version(options)
             if v:
-                clog().info("Interpreter version: %s", v)
+                v = '.'.join(map(str, v))
+                clog().info("%s's version: (%s)", repr(self), v)
 
         spawner = PopenSpawnExt if subprocess else pexpect.spawn
         try:
@@ -968,13 +969,16 @@ def _get_version(self, options):
             return None
 
         try:
-            out = subprocess.check_output(cmd).decode(self.encoding)
+            out = subprocess.check_output(cmd,
+                                          stderr=subprocess.STDOUT).decode(
+                                              self.encoding
+                                          )
             version = self._parse_version(out)
 
         except Exception as err:
             clog().warn(
-                "Failed to obtain runner version (%s).\nExecuted command: %s",
-                str(err), ' '.join(cmd)
+                "Failed to obtain %s's version (%s).\nExecuted command: %s",
+                repr(self), str(err), ' '.join(cmd)
             )
             return None
 

docs/languages/ruby.md

@@ -21,7 +21,7 @@ and the underlying runner or interpreter:
 <!-- matrix CI begin -->
 
 | Language   | Runner/Interpreter   |
-|:----------:|:--------------------:|
+|------------|----------------------|
 | 2.4        | 0.9.6                |
 | 2.5        | 0.9.6                |
 | 2.6        | 1.0.0                |

docs/overview/faq.md

@@ -124,7 +124,7 @@ of Python examples you can do:
 $ byexample -x-log-mask byexample.exec.python:chat -l python test/ds/db-stock-model
 [i:exec.python] Initializing Python Runner
 [i:exec.python] Spawn command line: /usr/bin/env python -i
-[i:exec.python] Interpreter version: <...>
+[i:exec.python] Python Runner's version: <...>
 ex:
     import sqlite3
 ex:
@@ -216,11 +216,10 @@ If you want to know the exact command line used by `byexample`, you can
 find it adding more verbosity:
 
 ```shell
-$ byexample -l python -x-shebang 'python:env python99' -v test/ds/db-stock-model   # byexample: +norm-ws
+$ byexample -l python -x-shebang 'python:env python99' -v test/ds/db-stock-model   # byexample: +norm-ws +diff=ndiff
 [i] Initializing Python Runner
 [i] Spawn command line: env python99
-<...>
-[w] Failed to obtain runner version <...>
+[w] Failed to obtain Python Runner's version <...>
 [w] Initialization of Python Runner failed.
 <...>
 ```

test/runner_version_matrix.py

@@ -0,0 +1,133 @@
+import sys, re, collections, pprint
+from tabulate import tabulate
+
+lang_re = re.compile(
+    r'''^Lang[ ]
+         (?P<language> \w+)[ ]
+         test[ ]?
+         (?:
+            \( (?P<lang_version>  [^)]+) \)  | [^(]
+         )
+         ''', re.VERBOSE)
+
+runner_re = re.compile(
+    r''' (?P<language> \w+)[ ]
+         Runner's[ ]version:[ ]
+         \( (?P<runner_version>  [^)]+) \)$
+         ''', re.VERBOSE)
+
+failed_re = re.compile(
+    r''' Failed[ ]to[ ]obtain[ ](?P<language> \w+)[ ]
+         Runner's[ ]version.*$
+         ''', re.VERBOSE)
+
+def parse_line(line):
+    '''
+        >>> parse_line(r"Lang Ruby test (3.1)  ... Ruby Runner's version: (1.4.1)")
+        ('Ruby', '3.1', '1.4.1')
+
+        >>> parse_line(r"Lang Ruby test ... Ruby Runner's version: (1.4.1)")
+        ('Ruby', 'latest', '1.4.1')
+
+        >>> parse_line(r"Lang Ruby test ... Failed to obtain Ruby Runner's version ...")
+        ('Ruby', 'latest', 'unknown (to review)')
+
+        >>> parse_line(r"Lang Ruby test ... Failed to obtain Python Runner's version ...")
+        None
+    '''
+
+    # "Lang Ruby test (3.1)  ... Ruby Runner's version: (1.4.1)"
+    # |--------------------|
+    m = lang_re.match(line)
+    assert m
+
+    language, lang_version = m.group('language', 'lang_version')
+    if not lang_version:
+        lang_version = 'latest'
+
+    # "Lang Ruby test (3.1)  ... Failed to obtain Ruby Runner's version ..."
+    #       ^^^^                |-----------------^^^^-----------------|
+    m = failed_re.search(line)
+    if m:
+        language2 = m.group('language')
+        if language == language2:
+            # this case means that something went wrong when byexample
+            # tried to retrieve the version of the runner
+            runner_version = '(in review)'
+            return language, lang_version, runner_version
+
+        # "Lang Ruby test (3.1)  ... Failed to obtain Python Runner's version ..."
+        #       ^^^^                |-----------------^^^^^^-----------------|
+        # In this case the failure is not a failure for our language of
+        # interest so we skip the line
+        return language, lang_version, None
+
+    # "Lang Ruby test (3.1)  ... Ruby Runner's version: (0.9.6)"
+    #       ^^^^                |^^^^--------------------------|
+    m = runner_re.search(line)
+    if not m:
+        return language, lang_version, None
+
+    language2, runner_version = m.group('language', 'runner_version')
+    if language != language2:
+        # Not our language of interest, skip the line
+        return language, lang_version, None
+
+    return language, lang_version, runner_version
+
+matrix = collections.defaultdict(dict)
+with open(sys.argv[1], 'rt') as f:
+    for line in f:
+
+        # If it is not a specific job for testing a language, skip it
+        if not line.startswith('Lang '):
+            continue
+
+        # If it is not the specific step that run the tests, skip it
+        if 'Run make lang-' not in line:
+            continue
+
+        language, lang_version, runner_version = parse_line(line)
+        if runner_version:
+            matrix[language][lang_version] = runner_version
+        else:
+            if lang_version not in matrix[language]:
+                matrix[language][lang_version] = 'unknown'
+
+begin_marker = '<!-- matrix CI begin -->'
+end_marker = '<!-- matrix CI end -->'
+
+document_files = {
+        'iasm': 'docs/languages/iasm.md',
+        'powershell': 'docs/languages/powershell.md',
+        'shell': 'docs/languages/shell.md',
+        'java': 'docs/languages/java.md',
+        'python': 'docs/languages/python.md',
+        'javascript': 'docs/languages/javascript.md',
+        'ruby': 'docs/languages/ruby.md',
+        }
+
+for language, data in matrix.items():
+    data = [[lang_ver, runner_ver] for lang_ver, runner_ver in data.items()]
+
+    doc_file = document_files[language.lower()]
+    with open(doc_file, 'rt') as f:
+        doc = f.read()
+
+    if begin_marker not in doc or end_marker not in doc:
+        raise Exception(f"Markers are missing in {doc_file}")
+
+    head, remain = doc.split(begin_marker, 1)
+    _, tail = remain.split(end_marker, 1)
+    del remain
+
+    table = tabulate(
+        data,
+        ["Language", "Runner/Interpreter"],
+        tablefmt='github',
+        disable_numparse=True
+        )
+
+    doc = head + begin_marker + '\n\n' + table + '\n\n' + end_marker + tail
+    with open(doc_file, 'wt') as f:
+        f.write(doc)