diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 34a59e5a..b8323a2e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -48,7 +48,7 @@ jobs: - name: Check for changed files id: check - uses: stackabletech/actions/detect-changes@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/detect-changes@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: patterns: | - '.github/workflows/build.yaml' @@ -164,7 +164,7 @@ jobs: - name: Build Container Image id: build - uses: stackabletech/actions/build-container-image@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/build-container-image@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-name: ${{ env.OPERATOR_NAME }} image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} @@ -173,7 +173,7 @@ jobs: - name: Publish Container Image to oci.stackable.tech if: ${{ !github.event.pull_request.head.repo.fork }} - uses: stackabletech/actions/publish-image@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/publish-image@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -184,7 +184,7 @@ jobs: - name: Publish Container Image to quay.io if: ${{ !github.event.pull_request.head.repo.fork }} - uses: stackabletech/actions/publish-image@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/publish-image@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build @@ -216,7 +216,7 @@ jobs: - name: Publish and Sign Image Index to oci.stackable.tech id: publish-oci - uses: stackabletech/actions/publish-image-index-manifest@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/publish-image-index-manifest@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -226,7 +226,7 @@ jobs: - name: Publish and Sign Image Index to quay.io id: publish-quay - uses: stackabletech/actions/publish-image-index-manifest@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/publish-image-index-manifest@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-registry-uri: quay.io image-registry-username: stackable+robot_sdp_github_action_build @@ -311,7 +311,7 @@ jobs: submodules: recursive - name: Package, Publish, and Sign Helm Chart to oci.stackable.tech - uses: stackabletech/actions/publish-helm-chart@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/publish-helm-chart@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: chart-registry-uri: oci.stackable.tech chart-registry-username: robot$sdp-charts+github-action-build @@ -323,7 +323,7 @@ jobs: publish-and-sign: ${{ !github.event.pull_request.head.repo.fork }} - name: Package, Publish, and Sign Helm Chart to quay.io - uses: stackabletech/actions/publish-helm-chart@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/publish-helm-chart@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: chart-registry-uri: quay.io chart-registry-username: stackable+robot_sdp_charts_github_action_build @@ -354,13 +354,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Run OpenShift Preflight Check for oci.stackable.tech - uses: stackabletech/actions/run-openshift-preflight@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/run-openshift-preflight@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} - name: Run OpenShift Preflight Check for quay.io - uses: stackabletech/actions/run-openshift-preflight@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/run-openshift-preflight@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: image-index-uri: quay.io/stackable/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} image-architecture: ${{ matrix.arch }} @@ -402,11 +402,20 @@ jobs: persist-credentials: false - name: Send Notification - uses: stackabletech/actions/send-slack-notification@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/send-slack-notification@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: publish-helm-chart-result: ${{ needs.publish-helm-chart.result }} publish-manifests-result: ${{ needs.publish-index-manifest.result }} build-result: ${{ needs.build-container-image.result }} + # Provenance is generated per registry, but the notification only has a + # single field for it, so the two results are collapsed into the worst + # one. 'failure' must be reported verbatim, otherwise the notification + # is not marked as failed. + generate-provenance-result: >- + ${{ (needs.provenance-oci.result == 'failure' || needs.provenance-quay.result == 'failure') && 'failure' + || (needs.provenance-oci.result == 'cancelled' || needs.provenance-quay.result == 'cancelled') && 'cancelled' + || (needs.provenance-oci.result == 'skipped' || needs.provenance-quay.result == 'skipped') && 'skipped' + || 'success' }} slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} channel-id: C07UG6JH44F # notifications-container-images type: container-image-build diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 6d54cb8e..de17f182 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -43,7 +43,7 @@ jobs: # TODO: Enable the scheduled runs which hard-code what profile to use - name: Run Integration Test id: test - uses: stackabletech/actions/run-integration-test@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/run-integration-test@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: replicated-api-token: ${{ secrets.REPLICATED_API_TOKEN }} test-mode-input: ${{ inputs.test-mode-input }} @@ -53,7 +53,7 @@ jobs: - name: Send Notification if: ${{ failure() || github.run_attempt > 1 }} - uses: stackabletech/actions/send-slack-notification@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + uses: stackabletech/actions/send-slack-notification@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: slack-token: ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }} failed-tests: ${{ steps.test.outputs.failed-tests }} diff --git a/.github/workflows/pr_prek.yaml b/.github/workflows/pr_prek.yaml index 16606c35..5932e4bb 100644 --- a/.github/workflows/pr_prek.yaml +++ b/.github/workflows/pr_prek.yaml @@ -28,7 +28,7 @@ jobs: persist-credentials: false submodules: recursive fetch-depth: 0 - - uses: stackabletech/actions/run-prek@a8af17a19bdcc3b5da0065f76e73827ba0c072ce # v0.16.0 + - uses: stackabletech/actions/run-prek@dc83bb926cc464f0f32454e934777116bd1c7768 # v0.16.3 with: rust: ${{ env.RUST_TOOLCHAIN_VERSION }} hadolint: ${{ env.HADOLINT_VERSION }} diff --git a/scripts/auto-retry-tests.py b/scripts/auto-retry-tests.py index db7645b3..153b7fc1 100755 --- a/scripts/auto-retry-tests.py +++ b/scripts/auto-retry-tests.py @@ -25,6 +25,7 @@ from contextlib import contextmanager from dataclasses import asdict, dataclass from datetime import datetime +from enum import Enum from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -43,6 +44,14 @@ class TestConstants: FAILED_TEST_PATTERN = r"--- FAIL: kuttl/harness/([^\s]+)" +class InitOutcome(Enum): + """Outcome of the initialization phase, i.e. what the retry logic should do next.""" + + RETRY = "retry" # There are failed tests to retry + DONE = "done" # Nothing left to do, the run succeeded + ABORTED = "aborted" # The run could not be completed (setup/infrastructure error) + + @dataclass class RuntimeHistory: """Tracks runtime history for a specific test.""" @@ -1047,7 +1056,7 @@ def create_test_summary( self.test_summaries[test_name] = summary return summary - def initialize_run_mode(self) -> bool: + def initialize_run_mode(self) -> InitOutcome: """Initialize the test run based on mode (resume, rerun, or fresh).""" # Check if we're resuming from a previous state if self.failed_tests: # This will be populated if we loaded state @@ -1055,7 +1064,7 @@ def initialize_run_mode(self) -> bool: print(f"Found {len(self.failed_tests)} failed tests to continue with:") for i, test in enumerate(self.failed_tests, 1): print(f" {i}. {test}") - return True + return InitOutcome.RETRY # Check if we're rerunning only failed tests elif self.args.rerun_failed: @@ -1066,7 +1075,7 @@ def initialize_run_mode(self) -> bool: # Run initial test suite return self._run_initial_test_suite() - def _load_failed_tests_for_rerun(self) -> bool: + def _load_failed_tests_for_rerun(self) -> InitOutcome: """Load failed tests for rerun mode.""" # Load failed tests from the specified state file failed_tests = self.state_manager.load_failed_tests_from_state( @@ -1075,7 +1084,7 @@ def _load_failed_tests_for_rerun(self) -> bool: if not failed_tests: print("āŒ No failed tests found to rerun") - return False + return InitOutcome.ABORTED print(f"Found {len(failed_tests)} failed tests to rerun:") for i, test in enumerate(failed_tests, 1): @@ -1083,9 +1092,9 @@ def _load_failed_tests_for_rerun(self) -> bool: # Store failed tests for the retry process self.failed_tests = failed_tests - return True + return InitOutcome.RETRY - def _run_initial_test_suite(self) -> bool: + def _run_initial_test_suite(self) -> InitOutcome: """Run the initial test suite.""" print("\nStep 1: Running initial full test suite...") initial_result = self.test_executor.run_single_test_suite( @@ -1095,7 +1104,7 @@ def _run_initial_test_suite(self) -> bool: if initial_result.success: print(" All tests passed on initial run!") self.report_generator.generate_and_save_final_report(self, self.start_time) - return False # No need to continue + return InitOutcome.DONE # No need to continue # Parse failed tests print("\nStep 2: Parsing failed tests...") @@ -1104,7 +1113,7 @@ def _run_initial_test_suite(self) -> bool: ) if not failed_tests: - print(" No failed tests found in output but run-tests exited with code 1") + print("āŒ No failed tests found in output but run-tests exited with code 1") print( " This indicates an infrastructure or setup issue that prevents tests from running" ) @@ -1112,7 +1121,7 @@ def _run_initial_test_suite(self) -> bool: " Check the log file for connection errors, missing dependencies, or cluster issues" ) print(f" Log file: {initial_result.log_file}") - return False + return InitOutcome.ABORTED print(f" Found {len(failed_tests)} failed tests:") for i, test in enumerate(failed_tests, 1): @@ -1121,7 +1130,7 @@ def _run_initial_test_suite(self) -> bool: # Store failed tests and initial result log for state persistence self.failed_tests = failed_tests self.initial_result_log = initial_result.log_file - return True + return InitOutcome.RETRY def execute_parallel_retries(self) -> Dict[str, List[TestResult]]: """Execute parallel retry attempts for failed tests.""" @@ -1263,8 +1272,14 @@ def run(self) -> int: print("=" * 46) # Initialize based on run mode - if not self.initialize_run_mode(): - return 0 # Early exit (e.g., all tests passed initially) + outcome = self.initialize_run_mode() + if outcome is InitOutcome.DONE: + return 0 # All tests passed initially + if outcome is InitOutcome.ABORTED: + # The test run never got to a point where retrying makes sense + # (e.g. release installation or test generation failed). + print("\nāŒ Test run aborted, see above for details.") + return 1 # Execute test retries if not self.execute_test_retries():