Merge pull request #1151 from aaroncameron-wk/conformance-suites-diff

tests/integration_tests/validation/run_conformance_suites.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
-import subprocess
 import sys
 from argparse import ArgumentParser, Namespace
+from pathlib import Path
+from typing import Any, Optional, TYPE_CHECKING
 
 from tests.integration_tests.download_cache import download_and_apply_cache, download_taxonomy_package
-from tests.integration_tests.validation.validation_util import get_conformance_suite_test_results, save_timing_file
 from tests.integration_tests.validation.conformance_suite_config import ConformanceSuiteConfig
 from tests.integration_tests.validation.conformance_suite_configs import (
     ALL_CONFORMANCE_SUITE_CONFIGS,
@@ -14,7 +14,9 @@
 from tests.integration_tests.validation.download_conformance_suites import (
     download_conformance_suite, extract_conformance_suite
 )
-from typing import Any, Optional, TYPE_CHECKING
+from tests.integration_tests.validation.validation_util import (
+    get_conformance_suite_test_results, save_timing_file, save_actual_results_file, CONFORMANCE_SUITE_EXPECTED_RESOURCES_DIRECTORY, save_diff_html_file
+)
 
 if TYPE_CHECKING:
     from _pytest.mark import ParameterSet
@@ -146,13 +148,15 @@ def run_conformance_suites(
     if test_option:
         for config in conformance_suite_configs:
             shards: list[int] = []
+            full_run = True
             if shard:
                 for part in shard.split(','):
                     if '-' in part:
                         start, end = part.split('-')
                         shards.extend(range(int(start), int(end) + 1))
                     else:
                         shards.append(int(part))
+                full_run = set(shards) == set(range(0, config.shards))
             results = get_conformance_suite_test_results(
                 config,
                 shards=shards,
@@ -163,6 +167,11 @@ def run_conformance_suites(
             )
             if log_to_file:
                 save_timing_file(config, results)
+                actual_results_path = save_actual_results_file(config, results)
+                if full_run:
+                    expected_results_path = CONFORMANCE_SUITE_EXPECTED_RESOURCES_DIRECTORY / Path(config.name).with_suffix('.csv')
+                    if expected_results_path.exists():
+                        save_diff_html_file(expected_results_path, actual_results_path, Path(f'conf-{config.name}-diff.html'))
             all_results.extend(results)
     return all_results
 

tests/integration_tests/validation/validation_util.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
+import csv
+import difflib
 import json
 import multiprocessing
 import os.path
 import statistics
-import tempfile
 import zipfile
 from collections import defaultdict
 from collections.abc import Generator
-from contextlib import ExitStack
 from contextlib import nullcontext
 from dataclasses import dataclass
 from heapq import heapreplace
-from pathlib import PurePath, PurePosixPath, Path
+from pathlib import PurePosixPath, Path
 from typing import Any, Callable, ContextManager, TYPE_CHECKING, cast
 from unittest.mock import patch
 
@@ -27,7 +27,8 @@
 
 
 original_normalize_url_function = WebCache.normalizeUrl
-CONFORMANCE_SUITE_TIMING_PATH_PREFIX = 'tests/resources/conformance_suites_timing'
+CONFORMANCE_SUITE_EXPECTED_RESOURCES_DIRECTORY = Path('tests/resources/conformance_suites_expected')
+CONFORMANCE_SUITE_TIMING_RESOURCES_DIRECTORY = Path('tests/resources/conformance_suites_timing')
 
 
 @dataclass(frozen=True)
@@ -367,7 +368,7 @@ def get_conformance_suite_test_results_without_shards(
 
 
 def load_timing_file(name: str) -> dict[str, float]:
-    path = Path(CONFORMANCE_SUITE_TIMING_PATH_PREFIX) / Path(name).with_suffix(".json")
+    path = CONFORMANCE_SUITE_TIMING_RESOURCES_DIRECTORY / Path(name).with_suffix('.json')
     if not path.exists():
         return {}
     with open(path) as file:
@@ -378,6 +379,43 @@ def load_timing_file(name: str) -> dict[str, float]:
         }
 
 
+def save_actual_results_file(config: ConformanceSuiteConfig, results: list[ParameterSet]) -> Path:
+    """
+    Saves a CSV file with format "(Full testcase variation ID),(Code)".
+    Each row represents a unique code actually triggered by a variation.
+    If an expected results file exists for the given conformance suite config,
+    the actual results file is then compared to the expected results file and an
+    HTML diff file is generated so that differences can be reviewed.
+    :param config: The conformance suite config associated with the given results.
+    :param results: The full set of results from a conformance suite run.
+    :return: Path to the saved file
+    """
+    rows = []
+    for result in results:
+        testcase_id = result.id
+        actual_codes = result.values[0].get('actual')  # type: ignore[union-attr]
+        for code in actual_codes:
+            rows.append((testcase_id, code))
+    output_filepath = Path(f'conf-{config.name}-actual.csv')
+    with open(output_filepath, 'w') as file:
+        writer = csv.writer(file)
+        writer.writerows(sorted(rows))
+    return output_filepath
+
+
+def save_diff_html_file(expected_results_path: Path, actual_results_path: Path, output_path: Path) -> None:
+    with open(expected_results_path) as file:
+        expected_rows = [row for row in file]
+    with open(actual_results_path) as file:
+        actual_rows = [row for row in file]
+    html = difflib.HtmlDiff().make_file(
+        expected_rows, actual_rows,
+        fromdesc='Expected', todesc='Actual', context=True, numlines=6
+    )
+    with open(output_path, 'w') as file:
+        file.write(html)
+
+
 def save_timing_file(config: ConformanceSuiteConfig, results: list[ParameterSet]) -> None:
     timing: dict[str, float] = defaultdict(float)
     for result in results: