Move PR builders and nightly test runners to Github Actions [API-713] (…

…#462)

* Move PR builders and nightly test runners to Github Actions

This PR introduces new workflows to run tests on pull requests,
pushes, and nightly.

With this change, we would run the following.

On PRs and pushes:

- Run tests with Python [ 2.7, 3.9 ] on [ Ubuntu, Windows ] and upload
results to Codecov
- Run black to verify code style
- Generate documentation to verify its correctness

Nightly:

- Run tests with Python [ 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 ] on
[ Ubuntu, Windows ]

Also, did some cleanups to have a consistent style among the workflow
definitions.

* upload rc logs

* disable installing nose from wheel

* revert back to wheel instalation for nosetests

* traverse all namespaces in the given directory for nosetests

* fix datetime tests for Windows

* flush stdout before starting the tests

* add missing comma to python version matrix in nightly runner

* address review comments

.github/workflows/jira_trigger.yml

@@ -1,40 +1,36 @@
 on:
   issues:
-    types: [labeled]
-
+    types: [ labeled ]
 name: Create Issue from Label
-
 jobs:
   build:
     if: ${{ github.event.label.name == 'to-jira' }}
     runs-on: ubuntu-latest
     name: Create Issue
     steps:
-     - name: Checkout
-       uses: actions/checkout@master
-
-     - name: Login
-       uses: atlassian/gajira-login@master
-       env:
-        JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
-        JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
-        JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
-
-     - name: Create
-       id: create
-       uses: atlassian/gajira-create@master
-       with:
-         project: API
-         issuetype: Bug
-         summary: | 
-          ${{ github.event.issue.title }} [GH-${{ github.event.issue.number }}]
-         description: |
-          *URL:* ${{ github.event.issue.html_url }}
-          *Created By:* ${{ github.event.issue.user.login }}
-          ${{ github.event.issue.body }}
-         fields: '{"labels": ["GitHub","Python"]}'
-     - name: Transition Issue
-       uses: atlassian/gajira-transition@master
-       with:
-        issue: ${{ steps.create.outputs.issue }}
-        transition: "To Do"
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: Login
+        uses: atlassian/gajira-login@master
+        env:
+          JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+          JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+          JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+      - name: Create
+        id: create
+        uses: atlassian/gajira-create@master
+        with:
+          project: API
+          issuetype: Bug
+          summary: |
+            ${{ github.event.issue.title }} [GH-${{ github.event.issue.number }}]
+          description: |
+            *URL:* ${{ github.event.issue.html_url }}
+            *Created By:* ${{ github.event.issue.user.login }}
+            ${{ github.event.issue.body }}
+          fields: '{"labels": ["GitHub","Python"]}'
+      - name: Transition Issue
+        uses: atlassian/gajira-transition@master
+        with:
+          issue: ${{ steps.create.outputs.issue }}
+          transition: "To Do"

.github/workflows/nightly_runner.yml

@@ -0,0 +1,41 @@
+name: Runs the tests nightly
+on:
+  schedule:
+    - cron: '0 2 * * *'
+jobs:
+  run-tests:
+    runs-on: ${{ matrix.os }}
+    name: Run tests with Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [ 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 ]
+        os: [ ubuntu-latest, windows-latest ]
+      fail-fast: false
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '8'
+      - name: Checkout to code
+        uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-test.txt
+      - name: Run tests
+        env:
+          HAZELCAST_ENTERPRISE_KEY: ${{ secrets.HAZELCAST_ENTERPRISE_KEY }}
+        run: python run_tests.py
+      - name: Upload remote controller logs on test failure
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: rc-logs-${{ matrix.python-version }}-${{ matrix.os }}
+          path: |
+            rc_stderr.log
+            rc_stdout.log

.github/workflows/test_runner.yml

@@ -0,0 +1,86 @@
+name: Run tests
+on:
+  push:
+    branches:
+      - master
+      - 4.*.z
+  pull_request:
+    branches:
+      - master
+      - 4.*.z
+jobs:
+  run-tests:
+    runs-on: ${{ matrix.os }}
+    name: Run tests with Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [ 2.7, 3.9 ]
+        os: [ ubuntu-latest, windows-latest ]
+      fail-fast: false
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: '8'
+      - name: Checkout to code
+        uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-test.txt
+      - name: Run tests
+        env:
+          HAZELCAST_ENTERPRISE_KEY: ${{ secrets.HAZELCAST_ENTERPRISE_KEY }}
+        run: python run_tests.py
+      - name: Upload remote controller logs on test failure
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: rc-logs-${{ matrix.python-version }}-${{ matrix.os }}
+          path: |
+            rc_stderr.log
+            rc_stdout.log
+      - name: Publish results to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          files: ./coverage.xml
+  run-linter:
+    runs-on: ubuntu-latest
+    name: Run black to check the code style
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Checkout to code
+        uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+      - name: Run black
+        run: |
+          black --check --config black.toml .
+  generate_docs:
+    runs-on: ubuntu-latest
+    name: Generate documentation
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Checkout to code
+        uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+      - name: Generate documentation
+        working-directory: docs
+        run: |
+          make html

run_tests.py

@@ -27,6 +27,8 @@ def wait_until_rc_is_ready():
             rc_process = start_rc(rc_stdout, rc_stderr)
             try:
                 wait_until_rc_is_ready()
+                sys.stdout.flush()
+
                 args = [
                     __file__,
                     "-v",
@@ -36,6 +38,7 @@ def wait_until_rc_is_ready():
                     "--cover-package=hazelcast",
                     "--cover-inclusive",
                     "--nologcapture",
+                    "--traverse-namespace",
                     "tests",
                 ]
 

tests/integration/backward_compatible/serialization/serializers_test.py

@@ -244,7 +244,20 @@ def test_datetime_date(self):
     def test_datetime_time(self):
         skip_if_client_version_older_than(self, "5.0")
         skip_if_server_version_older_than(self, self.client, "5.0")
-        value = datetime.datetime.now().time()
+        value = datetime.datetime.now()
+        if value.microsecond % 1000 == 0:
+            # A little hack for Windows. Time is precise to the
+            # milliseconds there. If we send the microseconds
+            # we have now, due to trailing zeros, get_from_server()
+            # call below will return the string representation with
+            # 3 digits for the microseconds. But, Python always expects
+            # 6 digits. So, the assertion will fail. To fix that,
+            # we add 1 microseconds to the value, so that in the Java
+            # side, nanoseconds representation will only have 3 trailing
+            # zeros, and will send the data as we want.
+            value = value + datetime.timedelta(microseconds=1)
+        value = value.time()
+
         self.map.set("key", value)
         self.assertEqual(value, self.map.get("key"))
         response = self.get_from_server()
@@ -254,10 +267,25 @@ def test_datetime_datetime(self):
         skip_if_client_version_older_than(self, "5.0")
         skip_if_server_version_older_than(self, self.client, "5.0")
         value = datetime.datetime.now(timezone(datetime.timedelta(seconds=1800)))
+        if value.microsecond % 1000 == 0:
+            # A little hack for Windows. Time is precise to the
+            # milliseconds there. If we send the microseconds
+            # we have now, due to trailing zeros, get_from_server()
+            # call below will return the string representation with
+            # 3 digits for the microseconds. But, Python always expects
+            # 6 digits. So, the assertion will fail. To fix that,
+            # we add 1 microseconds to the value, so that in the Java
+            # side, nanoseconds representation will only have 3 trailing
+            # zeros, and will send the data as we want.
+            value = value + datetime.timedelta(microseconds=1)
+
         self.map.set("key", value)
         self.assertEqual(value, self.map.get("key"))
         response = self.get_from_server()
-        self.assertEqual(response, value.strftime("%Y-%m-%dT%H:%M:%S.%f+00:30"))
+        expected = value.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
+        # Java sends offset string with a : in between hour and minute
+        expected = "%s:%s" % (expected[:-2], expected[-2:])
+        self.assertEqual(response, expected)
 
     def test_bool_from_server(self):
         self.assertTrue(self.set_on_server("true"))