Merge pull request #84 from trunk-io/eli/behave-retry

Behave automatic retry

.github/workflows/rerun.yaml

@@ -0,0 +1,19 @@
+name: re-run workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        required: true
+jobs:
+  rerun:
+    runs-on: ubuntu-latest
+    steps:
+      - name: rerun ${{ inputs.run_id }}
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+          GH_DEBUG: api
+        run: |
+          gh run watch ${{ inputs.run_id }} > /dev/null 2>&1
+          gh run rerun ${{ inputs.run_id }} --failed

.github/workflows/retry-tests.yaml

@@ -0,0 +1,81 @@
+name: Run Behave Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: [self-hosted, linux]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run behave tests
+        run: behave --junit-directory=python/results --junit python/behave
+
+      - name: Flaky Test Reporter
+        if: ${{ always() }}
+        uses: trunk-io/analytics-uploader@main
+        with:
+          # Path to your test results.
+          junit-paths: "**/python/results/*-behave.xml"
+          # Provide your Trunk organization slug.
+          org-slug: trunk-staging-org
+          # Provide your Trunk API token as a GitHub secret.
+          token: ${{ secrets.TRUNK_STAGING_ORG_API_TOKEN }}
+        env:
+          TRUNK_API_ADDRESS: https://api.trunk-staging.io/v1/metrics/createBundleUpload
+        continue-on-error: true
+
+  # # Investigate how to get this working - seems like gh run rerun doesn't work from inside that exact run. Maybethis information could be passed along to another
+  # # workflow
+  # retry-failed-test:
+  #   runs-on: [self-hosted, linux]
+  #   needs: [test]
+  #   if: failure()
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: trunk install
+  #       uses: trunk-io/trunk-action/install@54ccfcf9add644a36a5aa1d0046c92f654ff9e45
+
+  #     - name: Get Current Job Log URL
+  #       uses: Tiryoh/gha-jobid-action@v1
+  #       id: jobs
+  #       with:
+  #         github_token: ${{ github.token }}
+  #         job_name: "test"
+
+  #     - name: Rerun failed jobs in the current workflow
+  #       env:
+  #         GH_TOKEN: ${{ github.token }}
+  #       run: gh run rerun --job ${{ steps.jobs.outputs.job_id }}
+
+  re-run:
+    runs-on: [self-hosted, linux]
+    needs: test
+    if: failure() && fromJSON(github.run_attempt) < 5
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: trunk install
+        uses: trunk-io/trunk-action/install@54ccfcf9add644a36a5aa1d0046c92f654ff9e45
+
+      - name: trigger job re-run
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+          GH_DEBUG: api
+        run: gh workflow run rerun.yml -F run_id=${{ github.run_id }}

.gitignore

@@ -62,4 +62,8 @@ rust-project.json
 **/junitresults*
 
 # virtual env
-venv
+venv
+
+# retry data created by tests
+retry.data
+TESTS-behave.xml

python/behave/behave.feature

@@ -0,0 +1,6 @@
+Feature: flaky test example written in behave
+
+  Scenario: run a simple test that fails until retried a third time
+     Given start the behave test
+      When read file - if its not there - create it and fail
+      Then file existed - verify it contains the string Hello World

python/behave/steps/retry.py

@@ -0,0 +1,37 @@
+import os
+import tempfile
+import time
+
+from behave import *
+
+temp_dir = tempfile.gettempdir()
+test_file = os.path.join(temp_dir, "retry.data")
+
+
+@given("start the behave test")
+def step_impl(context):
+    time.sleep(0.2)  # sleep for 500 milliseconds
+
+
+@when("read file - if its not there - create it and fail")
+def step_impl(context):
+    time.sleep(0.3)  # sleep for 300 milliseconds
+    if not os.path.exists(test_file):
+        with open(test_file, "w") as f:
+            f.write("")
+        raise AssertionError("File was not there, created it and failed")
+
+
+@then("file existed - verify it contains the string Hello World")
+def step_impl(context):
+    time.sleep(0.5)  # sleep for 500 milliseconds
+    with open(test_file, "r") as f:
+        contents = f.read()
+    if "Hello World" not in contents:
+        with open(test_file, "a") as f:
+            f.write("Hello World")
+        raise AssertionError(
+            "File did not contain the string Hello World, added it and failed"
+        )
+    # file existed - everything is good (delete it)
+    os.remove(test_file)

requirements.txt

@@ -1,3 +1,4 @@
 pytest
 robotframework
-robotframework-pabot
+robotframework-pabot
+behave