Merge 1cc87b7 into 9612388

.github/scripts/flake.py

@@ -0,0 +1,96 @@
+#  Copyright Amazon.com Inc. or its affiliates.
+#  SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+import subprocess
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+
+from agithub.GitHub import GitHub
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cmd', type=str, help='Command to run')
+    parser.add_argument('-i', type=int, help='Iterations')
+    parser.add_argument('--token', type=str, help='GitHub token')
+    parser.add_argument('-ff', action="store_true", help='Fail fast. If enabled, quit '
+                                                         'after the first failure')
+    args = parser.parse_args()
+
+    command = args.cmd
+    iterations = args.i
+    token = args.token
+    results = defaultdict(lambda: defaultdict(list))
+
+    for i in range(0, iterations):
+        print(f"Running iteration {i + 1} of {iterations}", flush=True)
+        process = subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
+        # If the tests failed, then we should check which test(s) failed in order to report it
+        if process.returncode != 0:
+            print(f"Iteration {i + 1} failed, saving and parsing results now", flush=True)
+            parse_test_results(i, results)
+            if args.ff:
+                break
+        else:
+            print("Succeeded with no failure", flush=True)
+
+    if len(results) > 0:
+        print("Found some flakiness. Creating/updating GitHub issue.", flush=True)
+        gh = GitHub(token=token)
+        existing_issues = gh.repos[os.getenv("GITHUB_REPOSITORY")].issues.get(creator="app/github-actions")
+        if existing_issues[0] == 200:
+            existing_issues = list(filter(lambda i: "[Bot] Flaky" in i["title"], existing_issues[1]))
+        else:
+            existing_issues = []
+
+        body = f"Flaky test(s) found for commit {os.getenv('GITHUB_SHA')}.\n" \
+               f" See the uploaded artifacts from the action for details.\n\n"
+        for test_class, v in results.items():
+            for test_case, failures in v.items():
+                body += f"- {test_class}.{test_case} failed {len(failures)} times over {iterations} iterations "
+                unique_failure_reasons = set(map(lambda f: f["failure"], failures))
+                body += f"with {len(unique_failure_reasons)} unique failures.\n"
+            body += "\n"
+
+        title = "[Bot] Flaky Test(s) Identified"
+        if existing_issues:
+            issue_number = existing_issues[0]["number"]
+            updated_issue = gh.repos[os.getenv("GITHUB_REPOSITORY")].issues[issue_number].patch(body={"body": body,
+                                                                                                      "title": title})
+            print(updated_issue, flush=True)
+        else:
+            issue = gh.repos[os.getenv("GITHUB_REPOSITORY")].issues.post(body={"body": body,
+                                                                               "title": title})
+            print(issue, flush=True)
+
+
+def parse_test_results(iteration, previous_results):
+    if not os.path.exists("target/surefire-reports"):
+        return
+    reports = list(filter(lambda f: f.startswith("TEST-") and f.endswith(".xml"), os.listdir(
+        "target/surefire-reports")))
+    for r in reports:
+        tree = ET.parse("target/surefire-reports/" + r)
+        for testcase in tree.getroot().findall("./testcase"):
+            failure = None
+            if testcase.find("failure") is not None:
+                failure = testcase.find("failure").text
+            elif testcase.find("error") is not None:
+                failure = testcase.find("error").text
+            if failure is not None:
+                previous_results[testcase.get("classname")][testcase.get("name")] \
+                    .append({"iteration": iteration, "failure": failure})
+                if not os.path.exists('failed_tests'):
+                    os.mkdir('failed_tests')
+                if testcase.find("system-out") is not None:
+                    with open(f'failed_tests/{iteration}-stdout.txt', 'w') as f:
+                        f.write(testcase.find("system-out").text)
+                if testcase.find("system-err") is not None:
+                    with open(f'failed_tests/{iteration}-stderr.txt', 'w') as f:
+                        f.write(testcase.find("system-err").text)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/flakeFinder.yaml

@@ -0,0 +1,27 @@
+name: Flake Finder
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - master
+
+jobs:
+  flake:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Analyze Test Run
+        run: >-
+          pip3 -q install agithub &&
+          python3 .github/scripts/flake.py --cmd "mvn -ntp -q verify" -i 10 --token "${{ github.token }}"
+      - name: Upload Errors
+        uses: actions/upload-artifact@v1.0.0
+        with:
+          name: Flaky test results
+          path: failed_tests/