Add testing for ReBench on macOS on GitHub Actions (#226)

.github/workflows/ci.yml

@@ -3,22 +3,24 @@ name: CI
 on: [push, pull_request]
 
 jobs:
-  build:
+  test-linux:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ '3.7', '3.11', 'pypy3.9' ]
+        # Don't test pypy3.10, it currently is broken on GitHub Actions.
+        # re-enable once PyYAML can be installed again. See #225
+        python-version: [ '3.7', '3.11' ]  # 'pypy3.10'
         yaml-parser: ['', 'ruamel']
         include:
           - python-version: 3.7
             coverage: "--cov=rebench"
         exclude:
           - python-version: 3.11
             yaml-parser: ruamel
-          - python-version: pypy3
+          - python-version: pypy3.10
             yaml-parser: ruamel
-    name: Python ${{ matrix.python-version }} ${{ matrix.yaml-parser }}
+    name: "Ubuntu-latest: Python ${{ matrix.python-version }} ${{ matrix.yaml-parser }}"
     steps:
       - name: Checkout ReBench
         uses: actions/checkout@v3
@@ -60,8 +62,31 @@ jobs:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
         if: ${{ matrix.coverage && env.COVERALLS_REPO_TOKEN != '' }}
 
+  test-macos:
+    runs-on: macos-latest
+    name: "macOS: Python 3.11"
+    steps:
+      - name: Checkout ReBench
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install PyTest
+        run: pip install pytest
+
+      - name: Install ReBench dependencies
+        run: pip install .
+
+      - name: Run tests
+        run: |
+          python -m pytest
+          (cd rebench && rebench ../rebench.conf e:TestRunner2)
+
   test-docker:
-    name: Test ReBench in Docker
+    name: "Docker: python:3"
     runs-on: ubuntu-latest
     container:
       image: python:3
@@ -70,9 +95,6 @@ jobs:
       - name: Check for dockerenv file
         run: (ls /.dockerenv && echo Found dockerenv) || (echo No dockerenv)
 
-      - name: Make Python to be Python3
-        run: ln -s /usr/bin/python3 /usr/bin/python && ln -s /usr/bin/pip3 /usr/bin/pip
-
       - name: Install Time Command
         run: |
           apt-get update
@@ -82,13 +104,13 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install PyTest
-        run: pip install pytest
+        run: pip3 install pytest
 
       - name: Install ReBench dependencies
-        run: pip install .
+        run: pip3 install .
 
       - name: Run Test Run
         run: (cd rebench && rebench -D ../rebench.conf e:TestRunner2)
 
       - name: Run Unit Tests
-        run: python -m pytest
+        run: python3 -m pytest

rebench/denoise.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import getpass
 import json
 import os

rebench/executor.py

@@ -17,8 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
-from __future__ import with_statement
-
 from codecs import open as open_with_enc
 from collections import deque
 from math import floor

rebench/rebench.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # ReBench is a tool to run and document benchmarks.
 #
 # It is inspired by JavaStats implemented by Andy Georges.

rebench/reporter.py

@@ -17,8 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
-from __future__ import with_statement
-
 from time import time
 from operator import itemgetter
 import json

rebench/subprocess_with_timeout.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 from select     import select
 from subprocess import PIPE, STDOUT, Popen
 from threading  import Thread, Condition

rebench/tests/bugs/issue_111_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating a failing benchmark
-from __future__ import print_function
-
 print("Bench: iterations=1 runtime: 1000ms")
 print("Bench: iterations=1 runtime: 1000ms")
 print("Bench: iterations=1 runtime: 10ms")

rebench/tests/bugs/issue_27_vm.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating a failing benchmark
-from __future__ import print_function
-
 print("Starting Richards benchmark ...")
 print("Results are incorrect")

rebench/tests/bugs/vm-one-result.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 import random
 

rebench/tests/environment_test.py

@@ -34,7 +34,7 @@ def test_environment(self):
 
         self.assertTrue('manualRun' in env)
         self.assertGreater(env['memory'], 0)
-        self.assertGreater(env['clockSpeed'], 0)
+        self.assertGreaterEqual(env['clockSpeed'], 0)
 
         self.assertGreaterEqual(len(env['software']), 3)
 

rebench/tests/features/ignore_timeouts_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 import random
 from time import sleep

rebench/tests/features/issue_15_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 import random
 

rebench/tests/features/issue_16_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 
 print(sys.argv)

rebench/tests/features/issue_19_one_data_point_test.py

@@ -17,8 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-from __future__ import print_function
-
 from ...configurator     import Configurator, load_config
 from ...executor         import Executor, RoundRobinScheduler
 from ...persistence      import DataStore

rebench/tests/features/issue_19_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 import random
 

rebench/tests/features/issue_31_multivariate_data_points_test.py

@@ -17,8 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-from __future__ import print_function
-
 from ...configurator           import Configurator, load_config
 from ...executor               import Executor
 from ...persistence            import DataStore

rebench/tests/features/issue_31_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 
 print(sys.argv)

rebench/tests/features/issue_34_vm.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-from __future__ import print_function
-
+#!/usr/bin/env python3
 import sys
 
 print(sys.argv)

rebench/tests/features/issue_42_test.py

@@ -17,7 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-from __future__ import print_function
 import os
 import unittest
 from unittest import skip

rebench/tests/features/issue_42_vm.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import random
 import os
 import sys
@@ -14,7 +12,8 @@
 print(env)
 
 known_envvars = ["PWD", "SHLVL", "VERSIONER_PYTHON_VERSION",
-                 "_", "__CF_USER_TEXT_ENCODING", "LC_CTYPE"]
+                 "_", "__CF_USER_TEXT_ENCODING", "LC_CTYPE",
+                 "CPATH", "LIBRARY_PATH", "MANPATH", "SDKROOT"]
 
 if test == "as-expected":
     if os.environ.get("IMPORTANT_ENV_VARIABLE", None) != "exists":

rebench/tests/features/issue_58_build_vm_test.py

@@ -17,7 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-from __future__ import print_function
 import os
 
 from ...configurator     import Configurator, load_config

rebench/tests/features/issue_59_build_suite_test.py

@@ -17,7 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-from __future__ import print_function
 import os
 
 from ...configurator     import Configurator, load_config

rebench/tests/features/issue_81_unicode_test.py

@@ -17,7 +17,6 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-from __future__ import print_function
 import os
 
 from codecs import open as open_with_enc

rebench/tests/interop/time_adapter_test.py

@@ -33,7 +33,7 @@ class TimeAdapterTest(TestCase):
     def test_acquire_command(self):
         adapter = TimeAdapter(False, None)
         cmd = adapter.acquire_command(_TestRunId())
-        self.assertTrue(cmd.startswith("/usr/bin/time"))
+        self.assertRegex(cmd, r"^/.*/bin/g?time")
 
     def test_parse_data(self):
         data = """real        11.00

rebench/tests/mock_http_server.py

@@ -4,24 +4,28 @@
 from threading import Thread
 
 
-_put_requests = 0
-
-
 class _RequestHandler(BaseHTTPRequestHandler):
     def do_GET(self):
         self.send_response(200)
         self.end_headers()
+        self.send_header("Content-Length", 0)
 
     def do_PUT(self):
         self.send_response(200)
+        self.send_header("Content-Length", 0)
         self.end_headers()
-        global _put_requests  # pylint: disable=global-statement
-        _put_requests += 1
+        self.server.put_requests += 1
 
     def log_request(self, code='-', size='-'):
         pass
 
 
+class HTTPServerWithCounter(HTTPServer):
+    def __init__(self, *args, **kwargs):
+        super(HTTPServerWithCounter, self).__init__(*args, **kwargs)
+        self.put_requests = 0
+
+
 class MockHTTPServer(object):
 
     def __init__(self):
@@ -39,10 +43,7 @@ def get_free_port(self):
         return port
 
     def start(self):
-        global _put_requests  # pylint: disable=global-statement
-        _put_requests = 0
-
-        self._server = HTTPServer(('localhost', self._port), _RequestHandler)
+        self._server = HTTPServerWithCounter(('localhost', self._port), _RequestHandler)
 
         self._thread = Thread(target=self._server.serve_forever)
         self._thread.daemon = True
@@ -51,9 +52,7 @@ def start(self):
     def shutdown(self):
         self._server.shutdown()
 
-    @staticmethod
-    def get_number_of_put_requests():
-        global _put_requests  # pylint: disable=global-statement
-        result = _put_requests
-        _put_requests = 0
+    def get_number_of_put_requests(self):
+        result = self._server.put_requests
+        self._server.put_requests = 0
         return result

rebench/tests/subprocess_timeout_test.py

@@ -53,7 +53,7 @@ def test_exec_with_timeout(self):
         self.assertEqual(None, err)
 
     def test_exec_with_timeout_python_interpreter(self):
-        cmdline = "python -c \"while True: pass\""
+        cmdline = "python3 -c \"while True: pass\""
         (return_code, output, err) = sub.run(cmdline, {}, cwd=self._path,
                                              stdout=subprocess.PIPE,
                                              stderr=subprocess.STDOUT,

rebench/tests/test-vm1.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
-
 import sys
 import random
 

rebench/tests/test-vm2.py

@@ -1,6 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # simple script emulating an executor generating benchmark results
-from __future__ import print_function
 import sys
 import random
 

setup.py

@@ -38,11 +38,11 @@
       packages=find_packages(exclude=['*.tests']),
       package_data={'rebench': ['rebench-schema.yml']},
       install_requires=[
-          'PyYAML>=3.12',
+          'PyYAML>=6.0',
           'pykwalify>=1.8.0',
-          'humanfriendly>=8.0',
+          'humanfriendly>=10.0',
           'py-cpuinfo==9.0.0',
-          'psutil>=5.6.7'
+          'psutil>=5.9.5'
       ],
       entry_points = {
           'console_scripts' : [