Add conversion and traceability for third-party test vectors

crypto/cipher_extra/aead_test.cc

@@ -1199,8 +1199,11 @@ TEST(AEADTest, WycheproofAESGCMSIV) {
                 });
 }
 
+//= third_party/vectors/vectors_spec.md#wycheproof
+//= type=test
+//# AWS-LC MUST test against `testvectors_v1/aes_gcm_test.txt`.
 TEST(AEADTest, WycheproofAESGCM) {
-  FileTestGTest("third_party/wycheproof_testvectors/aes_gcm_test.txt",
+  FileTestGTest("third_party/vectors/converted/wycheproof/testvectors_v1/aes_gcm_test.txt",
                 [](FileTest *t) {
                   std::string key_size_str;
                   ASSERT_TRUE(t->GetInstruction(&key_size_str, "keySize"));

sources.cmake

@@ -289,11 +289,11 @@ set(
   crypto/x509/test/trailing_data_leaf_name_constraints.pem
   crypto/x509/test/trailing_data_leaf_subject_alt_name.pem
   crypto/x509/test/trailing_data_leaf_subject_key_identifier.pem
+  third_party/vectors/converted/wycheproof/testvectors_v1/aes_gcm_test.txt
   third_party/wycheproof_testvectors/aes_cbc_pkcs5_test.txt
   third_party/wycheproof_testvectors/aes_ccm_test.txt
   third_party/wycheproof_testvectors/aes_cmac_test.txt
   third_party/wycheproof_testvectors/aes_gcm_siv_test.txt
-  third_party/wycheproof_testvectors/aes_gcm_test.txt
   third_party/wycheproof_testvectors/chacha20_poly1305_test.txt
   third_party/wycheproof_testvectors/dsa_test.txt
   third_party/wycheproof_testvectors/ecdh_secp224r1_test.txt

third_party/vectors/.duvet/.gitignore

@@ -0,0 +1,2 @@
+reports/
+requirements/

third_party/vectors/.duvet/config.toml

@@ -0,0 +1,13 @@
+'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json"
+
+[[source]]
+pattern = "../../crypto/**/*test.cc"
+
+[[specification]]
+source = "vectors_spec.md"
+
+[report.html]
+enabled = true
+
+[report.snapshot]
+enabled = true

third_party/vectors/.duvet/snapshot.txt

@@ -0,0 +1,7 @@
+SPECIFICATION: [Test Vector Specification](vectors_spec.md)
+  SECTION: [wycheproof](#wycheproof)
+    TEXT[test]: AWS-LC MUST test against `testvectors_v1/aes_gcm_test.txt`.
+
+SPECIFICATION: [Test Vector Specification](vectors_spec.md)
+  SECTION: [wycheproof](#wycheproof)
+    TEXT[!MUST]: AWS-LC MUST test against `testvectors_v1/aes_gcm_test.txt`.

third_party/vectors/README.md

@@ -6,8 +6,8 @@ This directory contains cryptographic test vectors from external sources for use
 
 - `upstream/` - Raw test vectors from third-party sources (checked in for transparency)
 - `converted/` - Test vectors converted to our `file_test.h` format (consumed by tests)
-- `vectors_spec.md` - Listing of included test vectors in [RFC2119](https://www.rfc-editor.org/rfc/rfc2119) format for use by the `duvet` tool
-- `sync.py` - Script that updates upstream test vectors, converts them to the `file_test.h` format, and updates the `vectors_spec.md` specification.
+- `vectors_spec.md` - Auto-generated specification listing all test vectors
+- `sync.py` - Script that updates upstream test vectors, converts them to the `file_test.h` format, and generates `vectors_spec.md`
 
 ## Workflow
 
@@ -28,3 +28,23 @@ The following command adds a new vector and re-runs sync:
 ./sync.py --new [SOURCE_NAME]/[UPSTREAM_REPO_PATH_TO_FILE] 
 ```
 The path needs to include the source name from `sources.toml` and the file path relative to the upstream repository, for example `wycheproof/testvectors_v1/aes_gcm_test.json`.
+
+## Requirements Traceability
+
+The `vectors_spec.md` file lists all test vectors using the MUST keyword. Test files include duvet annotations linking them to the specification:
+
+```cpp
+//= third_party/vectors/vectors_spec.md#wycheproof
+//= type=test
+//# AWS-LC MUST test against `testvectors_v1/aes_gcm_test.txt`.
+TEST(AEADTest, WycheproofAESGCM) { ... }
+```
+
+The [duvet](https://github.com/awslabs/duvet) tool tracks which test vectors are used and where. It verifies that annotations haven't been removed, but does not verify test coverage. This helps document which vectors we use and maintain traceability.
+
+Duvet runs automatically when `sync.py` executes. To manually verify:
+
+```bash
+cd third_party/vectors
+duvet report --ci
+```

third_party/vectors/sync.py

@@ -10,20 +10,21 @@
 import shutil
 import filecmp
 
-from vectorslib import utils
+from vectorslib import utils, convert_vector, generate_spec
 from vectorslib.utils import SyncError
 
 
 def fetch_sources(
     clone_dir: pathlib.Path,
     sources: dict,
-    using_custom_clone_dir: bool = False,
+    reuse_existing: bool,
 ):
     for source_name, source_info in sources.items():
         source_clone_dir = clone_dir / source_name
 
+        # Reuse existing clone if present, otherwise clone fresh
         if source_clone_dir.is_dir():
-            assert using_custom_clone_dir
+            assert reuse_existing  # Should only happen with --clone-dir, not with new temp dir
             utils.warning(
                 f"using existing, potentially stale upstream clone of {source_name} at {source_clone_dir}"
             )
@@ -55,11 +56,8 @@ def update_sources(
     sources: dict,
     new_file: typing.Optional[str],
 ):
-    upstream_dir = cwd / "upstream"
-    upstream_dir.mkdir(parents=True, exist_ok=True)
-
+    # Ensure upstream directories exist
     for source_name, source_info in sources.items():
-        source_info["upstream_path"] = upstream_dir / source_name
         source_info["upstream_path"].mkdir(parents=True, exist_ok=True)
 
     # Add new file first to catch invalid file names and sources early
@@ -119,36 +117,81 @@ def convert_sources(
     clone_dir: pathlib.Path,
     sources: dict,
 ):
-    condir = cwd / "converted"
-    condir.mkdir(parents=True, exist_ok=True)
+    converted_dir = cwd / "converted"
+    converted_dir.mkdir(parents=True, exist_ok=True)
+
     for source_name, source_info in sources.items():
-        source_info["converted_path"] = condir / source_name
+        source_info["converted_path"] = converted_dir / source_name
         source_info["converted_path"].mkdir(parents=True, exist_ok=True)
-        assert source_info["converted_path"].is_dir()
 
-    utils.warning("convert_sources isn't yet fully implemented")
+    for source_name, source_info in sources.items():
+        upstream_path = source_info["upstream_path"]
+        converted_path = source_info["converted_path"]
+
+        for upstream_file in upstream_path.rglob("*.json"):
+            relative_path = upstream_file.relative_to(upstream_path)
+            converted_file = converted_path / relative_path.with_suffix(".txt")
+
+            converted_file.parent.mkdir(parents=True, exist_ok=True)
+
+            try:
+                convert_vector.convert_file(upstream_file, converted_file)
+                utils.info(f"converted {source_name}/{relative_path}")
+            except Exception as e:
+                error_msg = f"failed to convert {source_name}/{relative_path}: {e}"
+                utils.error(error_msg)
+                raise SyncError(error_msg)
+
+
+def generate_and_verify_spec(
+    cwd: pathlib.Path,
+    sources: dict,
+):
+    generate_spec.write_spec(cwd, sources)
+    utils.info("generated vectors_spec.md")
+
+    duvet_result = subprocess.run(
+        ["duvet", "report", "--ci"],
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+    )
+    if duvet_result.returncode != 0:
+        utils.error("duvet verification failed")
+        utils.error(duvet_result.stderr)
+        raise SyncError("duvet verification failed")
+    utils.info("duvet verification passed")
 
 
 def sync_sources(
     cwd: pathlib.Path,
     clone_dir: pathlib.Path,
     sources: dict,
-    new_file: typing.Optional[str],
-    skip_update: bool,
-    skip_convert: bool,
-    using_custom_clone_dir: bool = False,
+    args: argparse.Namespace,
 ):
-    if not skip_update:
-        fetch_sources(clone_dir, sources, using_custom_clone_dir)
-        update_sources(cwd, sources, new_file)
+    # Set up directory paths that other phases depend on
+    upstream_dir = cwd / "upstream"
+    for source_name, source_info in sources.items():
+        source_info["upstream_path"] = upstream_dir / source_name
+        source_info["local_path"] = clone_dir / source_name
+
+    if not args.skip_update:
+        reuse_existing = args.clone_dir is not None
+        fetch_sources(clone_dir, sources, reuse_existing)
+        update_sources(cwd, sources, args.new)
     else:
         utils.info("skipping update")
 
-    if not skip_convert:
+    if not args.skip_convert:
         convert_sources(cwd, clone_dir, sources)
     else:
         utils.info("skipping convert")
 
+    if not args.skip_spec:
+        generate_and_verify_spec(cwd, sources)
+    else:
+        utils.info("skipping spec generation")
+
 
 def main() -> int:
     cwd = pathlib.Path.cwd()
@@ -184,6 +227,11 @@ def main() -> int:
         action="store_true",
         help="skip converting vectors to file_test.h format",
     )
+    parser.add_argument(
+        "--skip-spec",
+        action="store_true",
+        help="skip generating vectors_spec.md and duvet verification",
+    )
     parser.add_argument(
         "--clone-dir",
         metavar="DIR",
@@ -198,26 +246,11 @@ def main() -> int:
         if args.clone_dir:
             clone_dir = pathlib.Path(args.clone_dir)
             clone_dir.mkdir(parents=True, exist_ok=True)
-            sync_sources(
-                cwd,
-                clone_dir,
-                sources,
-                args.new,
-                args.skip_update,
-                args.skip_convert,
-                using_custom_clone_dir=True,
-            )
+            sync_sources(cwd, clone_dir, sources, args)
         else:
             with tempfile.TemporaryDirectory() as temp_clone_dir:
                 clone_dir = pathlib.Path(temp_clone_dir)
-                sync_sources(
-                    cwd,
-                    clone_dir,
-                    sources,
-                    args.new,
-                    args.skip_update,
-                    args.skip_convert,
-                )
+                sync_sources(cwd, clone_dir, sources, args)
     except SyncError as e:
         utils.error(str(e))
         return 1

third_party/vectors/vectors_spec.md

@@ -0,0 +1,8 @@
+# Test Vector Specification
+
+*This file is generated by sync.py. Do not edit by hand.*
+
+## wycheproof
+
+AWS-LC MUST test against `testvectors_v1/aes_gcm_test.txt`.
+

third_party/vectors/vectorslib/__init__.py

@@ -1 +1 @@
-__all__ = ["utils"]
+__all__ = ["utils", "convert_vector", "generate_spec"]