diff --git a/.github/scripts/check-hive-results.sh b/.github/scripts/check-hive-results.sh index d07efd1afcd..55a1160b21a 100755 --- a/.github/scripts/check-hive-results.sh +++ b/.github/scripts/check-hive-results.sh @@ -10,6 +10,21 @@ if ! command -v jq >/dev/null 2>&1; then exit 1 fi +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to process Hive client logs but was not found in PATH" + exit 1 +fi + +slugify() { + local input="${1:-}" + local lowered trimmed + lowered="$(printf '%s' "${input}" | tr '[:upper:]' '[:lower:]')" + trimmed="$(printf '%s' "${lowered}" | sed -E 's/[^a-z0-9._-]+/-/g')" + trimmed="${trimmed#-}" + trimmed="${trimmed%-}" + printf '%s' "${trimmed}" +} + results_dir="${1:-src/results}" if [ ! -d "$results_dir" ]; then @@ -86,15 +101,13 @@ for json_file in "${json_files[@]}"; do { echo "### Hive failures: ${suite_name:-$(basename "${json_file}" .json)}" printf '%s\n' "${failure_list}" + echo "Note: Hive scenarios may include multiple ethrex clients, so each failing case can have more than one log snippet." echo } >> "${GITHUB_STEP_SUMMARY}" fi suite_slug_raw="${suite_name:-$(basename "${json_file}" .json)}" - suite_slug="$(printf '%s' "${suite_slug_raw}" | tr '[:upper:]' '[:lower:]')" - suite_slug="$(printf '%s' "${suite_slug}" | sed -E 's/[^a-z0-9._-]+/-/g')" - suite_slug="${suite_slug#-}" - suite_slug="${suite_slug%-}" + suite_slug="$(slugify "${suite_slug_raw}")" suite_dir="${failed_logs_root}/${suite_slug:-suite}" mkdir -p "${suite_dir}" @@ -178,6 +191,198 @@ for json_file in "${json_files[@]}"; do done <<< "${suite_logs_output}" fi + client_case_entries="$( + jq -r ' + .testCases + | to_entries[] + | select(.value.summaryResult.pass != true) + | . as $case_entry + | ($case_entry.value.clientInfo? // {}) | to_entries[] + | [ + .value.logFile // "", + ($case_entry.value.name // ("case-" + $case_entry.key)), + $case_entry.key, + ($case_entry.value.start // ""), + ($case_entry.value.end // ""), + .key + ] + | @tsv + ' "${json_file}" 2>/dev/null || true + )" + generated_client_snippets=0 + if [ -n "${client_case_entries}" ]; then + client_logs_dir="${suite_dir}/client_logs" + mkdir -p "${client_logs_dir}" + + while IFS= read -r client_entry; do + [ -n "${client_entry}" ] || continue + IFS=$'\t' read -r client_log_rel raw_case_name case_id case_start case_end client_id <<< "${client_entry}" + + if [ -z "${client_log_rel}" ] || [ -z "${case_start}" ] || [ -z "${case_end}" ]; then + continue + fi + + log_copy_path="${suite_dir}/${client_log_rel}" + if [ ! -f "${log_copy_path}" ]; then + continue + fi + + case_slug="$(slugify "${raw_case_name}")" + if [ -n "${case_slug}" ]; then + case_slug="${case_slug}-case-${case_id}" + else + case_slug="case-${case_id}" + fi + + client_slug="$(slugify "${client_id}")" + if [ -z "${client_slug}" ]; then + client_slug="client" + fi + + case_dir="${client_logs_dir}/${case_slug}" + mkdir -p "${case_dir}" + snippet_path="${case_dir}/${client_slug}.log" + + python3 - "${log_copy_path}" "${snippet_path}" "${raw_case_name}" "${case_start}" "${case_end}" "${client_id}" "${client_log_rel}" <<'PY' +import sys +from datetime import datetime, timedelta +from pathlib import Path + +FORMATS = ("%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ") +CONTEXT_SECONDS = 2 +PREFETCH_LIMIT = 50 + +def normalise_timestamp_str(value): + if not value or not value.endswith("Z"): + return value + prefix = value[:-1] + if "." not in prefix: + return value + base, frac = prefix.split(".", 1) + frac_digits = "".join(ch for ch in frac if ch.isdigit()) + if not frac_digits: + return f"{base}.000000Z" + frac_digits = (frac_digits + "000000")[:6] + return f"{base}.{frac_digits}Z" + +def parse_timestamp(value): + if not value: + return None + value = normalise_timestamp_str(value) + for fmt in FORMATS: + try: + return datetime.strptime(value, fmt) + except ValueError: + continue + return None + +def timestamp_from_line(line): + if not line: + return None + token = line.split(" ", 1)[0] + if not token or not token[0].isdigit(): + return None + token = normalise_timestamp_str(token) + for fmt in FORMATS: + try: + return datetime.strptime(token, fmt) + except ValueError: + continue + return None + +log_path = Path(sys.argv[1]) +output_path = Path(sys.argv[2]) +case_name = sys.argv[3] +case_start_raw = sys.argv[4] +case_end_raw = sys.argv[5] +client_id = sys.argv[6] or "unknown" +client_log_rel = sys.argv[7] + +try: + log_content = log_path.read_text(encoding="utf-8", errors="replace").splitlines(keepends=True) +except Exception as exc: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(f"# Failed to read log '{log_path}': {exc}\n", encoding="utf-8") + sys.exit(0) + +start_ts = parse_timestamp(case_start_raw) +end_ts = parse_timestamp(case_end_raw) + +fallback_reason = None +if not start_ts or not end_ts or end_ts < start_ts: + fallback_reason = "Unable to determine reliable time window from test metadata." +else: + start_ts = start_ts - timedelta(seconds=CONTEXT_SECONDS) + end_ts = end_ts + timedelta(seconds=CONTEXT_SECONDS) + +captured_lines = [] +prefetch = [] +current_ts = None +capturing = False + +if not fallback_reason: + for line in log_content: + ts = timestamp_from_line(line) + if ts is not None: + current_ts = ts + + if not capturing: + prefetch.append(line) + if len(prefetch) > PREFETCH_LIMIT: + prefetch.pop(0) + + in_window = current_ts is not None and start_ts <= current_ts <= end_ts + + if in_window: + if not capturing: + captured_lines.extend(prefetch) + capturing = True + captured_lines.append(line) + elif capturing and current_ts is not None and current_ts > end_ts: + break + elif capturing: + captured_lines.append(line) + + if not captured_lines: + fallback_reason = "No timestamped log lines matched the computed time window." + +if fallback_reason: + captured_lines = log_content + +header_lines = [ + f"# Test: {case_name}\n", + f"# Client ID: {client_id}\n", + f"# Source log: {client_log_rel}\n", +] + +if start_ts and end_ts and not fallback_reason: + header_lines.append( + f"# Time window (UTC): {case_start_raw} .. {case_end_raw} (with ±{CONTEXT_SECONDS}s context)\n" + ) +else: + header_lines.append("# Time window (UTC): unavailable\n") + +if fallback_reason: + header_lines.append(f"# NOTE: {fallback_reason}\n") + +header_lines.append("\n") + +output_path.parent.mkdir(parents=True, exist_ok=True) +with output_path.open("w", encoding="utf-8") as dst: + dst.writelines(header_lines) + dst.writelines(captured_lines) +PY + + if [ -s "${snippet_path}" ]; then + generated_client_snippets=$((generated_client_snippets + 1)) + fi + done <<< "${client_case_entries}" + fi + + if [ "${generated_client_snippets}" -gt 0 ]; then + echo "Generated ${generated_client_snippets} client log snippet(s) in ${client_logs_dir}" + fi + echo "Saved Hive failure artifacts to ${suite_dir}" failures=$((failures + failed_cases)) diff --git a/.github/workflows/pr-main_l1.yaml b/.github/workflows/pr-main_l1.yaml index 0c0d1ed934e..15691741c79 100644 --- a/.github/workflows/pr-main_l1.yaml +++ b/.github/workflows/pr-main_l1.yaml @@ -146,55 +146,35 @@ jobs: simulation: ethereum/rpc-compat # https://github.com/ethereum/execution-apis/pull/627 changed the simulation to use a pre-merge genesis block, so we need to pin to a commit before that buildarg: "branch=d08382ae5c808680e976fce4b73f4ba91647199b" - hive_repository: ethereum/hive - hive_version: 7709e5892146c793307da072e1593f48039a7e4b artifact_prefix: rpc_compat - name: "Devp2p tests" simulation: devp2p limit: discv4|eth|snap/Ping|Findnode/WithoutEndpointProof|Findnode/PastExpiration|Amplification|Status|StorageRanges|ByteCodes|GetBlockHeaders|SimultaneousRequests|SameRequestID|ZeroRequestID|GetBlockBodies|MaliciousHandshake|MaliciousStatus|NewPooledTxs|GetBlockReceipts|BlockRangeUpdate|GetTrieNodes # Findnode/BasicFindnode fails due to packets being processed out of order # Findnode/UnsolicitedNeighbors flaky in CI very occasionally. When fixed replace all "Findnode/" with "Findnode" - hive_repository: ethereum/hive - hive_version: 7709e5892146c793307da072e1593f48039a7e4b artifact_prefix: devp2p - name: "Engine Auth and EC tests" simulation: ethereum/engine limit: engine-(auth|exchange-capabilities)/ - hive_repository: ethereum/hive - hive_version: 7709e5892146c793307da072e1593f48039a7e4b artifact_prefix: engine_auth_ec - # - name: "Cancun Engine tests" - # simulation: ethereum/engine - # limit: "engine-cancun" - # hive_repository: ethereum/hive - # hive_version: 7709e5892146c793307da072e1593f48039a7e4b - # artifact_prefix: engine_cancun + - name: "Cancun Engine tests" + simulation: ethereum/engine + limit: "engine-cancun" + artifact_prefix: engine_cancun - name: "Paris Engine tests" simulation: ethereum/engine limit: "engine-api" - hive_repository: ethereum/hive - hive_version: 7709e5892146c793307da072e1593f48039a7e4b artifact_prefix: engine_paris - name: "Engine withdrawal tests" simulation: ethereum/engine limit: "engine-withdrawals/Corrupted Block Hash Payload|Empty Withdrawals|engine-withdrawals test loader|GetPayloadBodies|GetPayloadV2 Block Value|Max Initcode Size|Sync after 2 blocks - Withdrawals on Genesis|Withdraw many accounts|Withdraw to a single account|Withdraw to two accounts|Withdraw zero amount|Withdraw many accounts|Withdrawals Fork on Block 1 - 1 Block Re-Org|Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload|Withdrawals Fork on Block 2|Withdrawals Fork on Block 3|Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload|Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org [^S]|Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org [^S]" - hive_repository: ethereum/hive - hive_version: 7709e5892146c793307da072e1593f48039a7e4b artifact_prefix: engine_withdrawals # Investigate this test # - name: "Sync" # simulation: ethereum/sync # limit: "" - # hive_repository: ethereum/hive - # hive_version: 7709e5892146c793307da072e1593f48039a7e4b # artifact_prefix: sync steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - tool-cache: false - large-packages: false - - name: Checkout sources uses: actions/checkout@v4 @@ -225,7 +205,7 @@ jobs: SIM_LIMIT: ${{ matrix.limit }} SIM_BUILDARG: ${{ matrix.buildarg }} run: | - FLAGS='--sim.parallelism 16 --sim.loglevel 1' + FLAGS='--sim.parallelism 4 --sim.loglevel 3' if [[ -n "$SIM_LIMIT" ]]; then escaped_limit=${SIM_LIMIT//\'/\'\\\'\'} FLAGS+=" --sim.limit '$escaped_limit'" @@ -239,8 +219,8 @@ jobs: id: run-hive-action uses: ethpandaops/hive-github-action@v0.5.0 with: - hive_repository: ${{ matrix.hive_repository }} - hive_version: ${{ matrix.hive_version }} + hive_repository: ethereum/hive + hive_version: 7709e5892146c793307da072e1593f48039a7e4b simulator: ${{ matrix.simulation }} client: ethrex client_config: ${{ steps.client-config.outputs.config }}