feat: added a workflow to be able to trigger all or priority failing …

…tests in the CI, given that tests don't get triggered intelligently at times (#28134)

.github/workflows/run-failing-tests.yml

@@ -0,0 +1,80 @@
+name: Run Failing Tests
+on:
+  workflow_dispatch:
+    inputs:
+      priority:
+        description: 'Priority:'
+        default: true
+        required: false
+
+permissions:
+  actions: read
+jobs:
+  display_test_results:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs:
+      - run_tests
+
+    steps:
+      - name: Download all test results
+        uses: actions/download-artifact@v3
+
+      - name: Combined Test Results
+        run: |
+          find . -name "test_results_*.txt" -exec cat {} + > combined_test_results.txt
+          echo "Test results summary:"
+          cat combined_test_results.txt
+
+  run_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+                  11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+                  21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+                  31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+                  41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
+                  51, 52, 53, 54, 55, 56, 57, 58, 59, 60,
+                  61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
+                  71, 72, 73, 74, 75, 76, 77, 78, 79, 80,
+                  81, 82, 83, 84, 85, 86, 87, 88, 89, 90,
+                  91, 92, 93, 94, 95, 96, 97, 98, 99, 100,
+                  101, 102, 103, 104, 105, 106, 107, 108, 109, 110,
+                  111, 112, 113, 114, 115, 116, 117, 118, 119, 120,
+                  121, 122, 123, 124, 125, 126, 127, 128 ]
+    steps:
+      - name: Checkout Ivy 🛎
+        uses: actions/checkout@v3
+        with:
+          path: ivy
+          persist-credentials: false
+          submodules: "recursive"
+          fetch-depth: 1
+
+      - name: Get Job URL
+        uses: Tiryoh/gha-jobid-action@v0
+        id: jobs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: ${{ github.job }}
+
+      - name: Setup Tests
+        run: |
+          pip3 install pymongo
+          cd ivy
+          mkdir .ivy
+          touch .ivy/key.pem
+          echo -n ${{ secrets.USER_API_KEY }} > .ivy/key.pem
+          python scripts/setup_tests/setup_failing_tests.py ${{ github.event.inputs.priority }} ${{ matrix.branch }}
+
+      - name: Run Tests
+        run: |
+          cd ivy
+          python scripts/run_tests/run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} 'false' 'false' ${{ github.run_id }} 'false' 'false' ${{ steps.jobs.outputs.html_url }}
+        continue-on-error: true
+
+      - name: Check on failures
+        if: steps.tests.outcome != 'success'
+        run: exit 1

scripts/setup_tests/setup_failing_tests.py

@@ -0,0 +1,120 @@
+from get_all_tests import BACKENDS
+from packaging import version
+from pymongo import MongoClient
+import requests
+import sys
+
+
+def get_latest_package_version(package_name):
+    try:
+        url = f"https://pypi.org/pypi/{package_name}/json"
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+        package_info = response.json()
+        versions = list(package_info["releases"].keys())
+        key = lambda x: version.parse(x)
+        return sorted(versions, key=key, reverse=True)
+    except requests.exceptions.RequestException:
+        print(f"Error: Failed to fetch package information for {package_name}.")
+        return None
+
+
+def main():
+    # connect to the database
+    priority = sys.argv[1] == "true"
+    run_iter = int(sys.argv[2]) - 1
+    cluster = MongoClient(
+        "mongodb+srv://readonly-user:hvpwV5yVeZdgyTTm@cluster0.qdvf8q3.mongodb.net"
+    )
+    ci_dashboard_db = cluster["ci_dashboard"]
+    ivy_tests_collection = ci_dashboard_db["ivy_tests"]
+    frontend_tests_collection = ci_dashboard_db["frontend_tests"]
+
+    # iterate over demos and collect ivy and frontend functions used
+    ivy_test_docs = ivy_tests_collection.find()
+    frontend_test_docs = frontend_tests_collection.find()
+    ivy_functions = [
+        ivy_test_doc["_id"]
+        for ivy_test_doc in ivy_test_docs
+        if not priority or ivy_test_doc.get("demos", None)
+    ]
+    frontend_functions = [
+        frontend_test_doc["_id"]
+        for frontend_test_doc in frontend_test_docs
+        if not priority or frontend_test_doc.get("demos", None)
+    ]
+    ivy_functions = sorted(list(set(ivy_functions)))
+    frontend_functions = sorted(list(set(frontend_functions)))
+    versions = {
+        backend: [
+            version_name.replace(".", "_")
+            for version_name in get_latest_package_version(backend)
+        ]
+        for backend in BACKENDS
+    }
+
+    # find corresponding test paths for those functions
+    ivy_test_paths = []
+    frontend_test_paths = []
+    for function in ivy_functions:
+        print("function", function)
+        result = ivy_tests_collection.find_one({"_id": function})
+        if result:
+            for backend in BACKENDS:
+                if backend in result:
+                    for version_name in versions[backend]:
+                        if version_name in result[backend]:
+                            if "status" in result[backend][version_name]:
+                                status = result[backend][version_name]["status"].get(
+                                    "cpu"
+                                )
+                                if not status and status is not None:
+                                    ivy_test_paths.append(
+                                        f"{result['test_path']},{backend}"
+                                    )
+                                break
+
+    for function in frontend_functions:
+        print("frontend function", function)
+        frontend = function.split(".")[0]
+        result = frontend_tests_collection.find_one({"_id": function})
+        if result and frontend in versions:
+            for frontend_version in versions[frontend]:
+                if frontend_version in result:
+                    backend_result = result[frontend_version]
+                    for backend in BACKENDS:
+                        if backend in backend_result:
+                            for version_name in versions[backend]:
+                                if version_name in backend_result[backend]:
+                                    if (
+                                        "status"
+                                        in backend_result[backend][version_name]
+                                    ):
+                                        status = backend_result[backend][version_name][
+                                            "status"
+                                        ].get("cpu")
+                                        if not status and status is not None:
+                                            frontend_test_paths.append(
+                                                f"{result['test_path']},{backend}"
+                                            )
+                                        break
+
+    all_tests = ivy_test_paths + frontend_test_paths
+    all_tests = [test_path.strip() for test_path in all_tests]
+    tests_per_run = 50
+    num_tests = len(all_tests)
+    start = run_iter * tests_per_run
+    end = (run_iter + 1) * tests_per_run
+    end = min(end, num_tests)
+    if start < end:
+        tests = all_tests[start:end]
+    else:
+        tests = []
+
+    # add those paths to the tests_to_run
+    with open("tests_to_run", "w") as write_file:
+        write_file.write("\n".join(tests))
+
+
+if __name__ == "__main__":
+    main()