fips205-source/tests/nist_acvp_vectors/extract_sha2_128s.py

#!/usr/bin/env python3
"""Deterministically re-derive tests/nist_acvp_vectors/SLH-DSA-sigVer-FIPS205/
sha2_128s_extracted.json from the official NIST ACVP-Server vector set.

Why this file exists at all: the ACVP vector file vendored in this repository
contains no SLH-DSA-SHA2-128s sigVer group, so the parameter set the associated
verification work is about had no NIST known-answer verification coverage. The
three 128s groups are therefore taken from upstream.

Why this SCRIPT exists: external review (round 7) observed that a hand-made
extraction with a prose provenance note is not auditable — the first version's
note said "only `sk` dropped" while three further fields had in fact been
removed. The transformation is now executable, pinned, and fails closed:

  * the upstream file's sha256 must match SOURCE_SHA256 exactly;
  * exactly EXPECTED_GROUPS groups must match the parameter set, each with
    EXPECTED_TESTS_PER_GROUP tests;
  * exactly one field, `sk`, is removed, and it must be present to be removed
    (so a schema change is caught, not silently transformed);
  * every other key is carried through untouched — BY CONSTRUCTION, which a
    reviewer verifies by reading `build()`, not by a self-check: no test inside a
    transformer can detect a corrupted input, since the transformer is what
    defines the output. The input is instead pinned by SOURCE_SHA256;
  * verify mode re-derives and byte-compares the committed file, so a hand-edit
    of the committed JSON IS caught;
  * output is canonical (fixed indent, trailing newline).

NOT A GATE — re-runnable EVIDENCE. Round-8 review noted the distinction: this
script needs network access, so nothing invokes it automatically (it is outside
`cargo test` and outside verification/check.sh). The committed JSON is still
trusted at review time; what this script provides is that a reviewer can
CHECK that trust cheaply and mechanically instead of taking a prose note's word.
A CI job running it in verify mode would close the remaining gap.

Usage:
    python3 extract_sha2_128s.py            # verify the committed file matches
    python3 extract_sha2_128s.py --write    # regenerate it

The upstream file is ~30 MB; it is fetched to a temporary path and not vendored.
"""

import argparse, hashlib, json, os, sys, tempfile, urllib.request

SOURCE_URL = (
    "https://raw.githubusercontent.com/usnistgov/ACVP-Server/master/"
    "gen-val/json-files/SLH-DSA-sigVer-FIPS205/internalProjection.json"
)
SOURCE_SHA256 = "a013fc2104f4ed4799d96d51141f65b965969b2cf10646626a021b6d456ce792"
PARAMETER_SET = "SLH-DSA-SHA2-128s"
EXPECTED_GROUPS = 3
EXPECTED_TESTS_PER_GROUP = 14
DROP_FIELDS = frozenset({"sk"})          # the ONLY per-test removal
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                   "SLH-DSA-sigVer-FIPS205", "sha2_128s_extracted.json")

PROVENANCE = {
    "what": f"{PARAMETER_SET} sigVer test groups, extracted verbatim from the "
            "official NIST ACVP-Server vector set.",
    "why": "The vector file vendored upstream in this directory contains NO "
           f"{PARAMETER_SET} sigVer group (only 192s/256f/SHAKE variants), so the "
           "one parameter set the verification work targets had zero NIST "
           "known-answer verification coverage.",
    "source_url": SOURCE_URL,
    "source_sha256": SOURCE_SHA256,
    "retrieved_utc": "2026-07-28",
    "extraction": "Produced by tests/nist_acvp_vectors/extract_sha2_128s.py, which "
                  "verifies the upstream sha256, selects every testGroup whose "
                  f"parameterSet == {PARAMETER_SET}, and removes exactly ONE "
                  "per-test field: `sk` (the private key, not needed to verify a "
                  "signature). Every other per-test field and all group and "
                  "top-level metadata are carried through unchanged by construction. "
                  "The guarantees that can actually fail are: the pinned upstream "
                  "SOURCE_SHA256; the requirement that `sk` be present to be dropped; "
                  "the expected group and per-group test counts; and verify mode, which "
                  "re-derives and byte-compares this committed file.",
    "note": "Test DATA only. Expected outcomes are NIST's `testPassed` field; "
            "`reason` records why a negative case must be rejected.",
    "regenerate": "python3 tests/nist_acvp_vectors/extract_sha2_128s.py --write",
}


def fetch() -> dict:
    with tempfile.NamedTemporaryFile(delete=False, suffix=".json") as fh:
        tmp = fh.name
    try:
        urllib.request.urlretrieve(SOURCE_URL, tmp)
        raw = open(tmp, "rb").read()
    finally:
        os.unlink(tmp)
    got = hashlib.sha256(raw).hexdigest()
    if got != SOURCE_SHA256:
        sys.exit(f"FATAL: upstream sha256 {got} != pinned {SOURCE_SHA256}.\n"
                 "The NIST file changed. Review the delta and update the pin "
                 "deliberately; do not regenerate blindly.")
    return json.loads(raw)


def build(full: dict) -> dict:
    groups = []
    for g in full["testGroups"]:
        if g.get("parameterSet") != PARAMETER_SET:
            continue
        tests = []
        for t in g["tests"]:
            for f in DROP_FIELDS:
                if f not in t:
                    sys.exit(f"FATAL: tcId {t.get('tcId')} has no field {f!r} to drop")
            # Fields are carried through BY CONSTRUCTION: `kept` is `t` minus
            # DROP_FIELDS, so every retained key holds the identical object.
            #
            # Round-8 review found a tautological `assert` here — it compared
            # `kept` against the comprehension that had just built it, so it could
            # never fire. The first attempt to repair it (comparing kept[k] to
            # t[k]) was tautological for the same reason, and that is the lesson
            # worth recording: NO check inside this function can detect a
            # corrupted input, because this function is what defines the output
            # from that input. Faithfulness here is a property of the two lines
            # below, which a reviewer reads; it is not something the script can
            # test about itself.
            #
            # What actually protects the result, and can fail:
            #   * SOURCE_SHA256 — the input is pinned, so upstream cannot drift
            #     or be substituted without an explicit, reviewed pin change;
            #   * the `f not in t` presence check above — `sk` must exist to be
            #     dropped, so a schema change is caught rather than silently
            #     producing a different transformation;
            #   * EXPECTED_GROUPS / EXPECTED_TESTS_PER_GROUP below;
            #   * verify mode, which re-derives and byte-compares the committed
            #     file, so a hand-edit of the committed JSON is caught.
            kept = {k: v for k, v in t.items() if k not in DROP_FIELDS}
            tests.append(kept)
        if len(tests) != EXPECTED_TESTS_PER_GROUP:
            sys.exit(f"FATAL: group {g['tgId']} has {len(tests)} tests, "
                     f"expected {EXPECTED_TESTS_PER_GROUP}")
        ng = {k: v for k, v in g.items() if k != "tests"}
        ng["tests"] = tests
        groups.append(ng)
    if len(groups) != EXPECTED_GROUPS:
        sys.exit(f"FATAL: found {len(groups)} {PARAMETER_SET} groups, expected {EXPECTED_GROUPS}")
    out = {"_provenance": PROVENANCE}
    for k, v in full.items():                    # all top-level metadata, verbatim
        if k != "testGroups":
            out[k] = v
    out["testGroups"] = groups
    return out


def render(obj: dict) -> str:
    return json.dumps(obj, indent=1) + "\n"


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--write", action="store_true", help="regenerate the committed file")
    args = ap.parse_args()
    text = render(build(fetch()))
    if args.write:
        open(OUT, "w").write(text)
        print(f"wrote {OUT} ({len(text)} bytes)")
        return 0
    if not os.path.exists(OUT):
        print(f"MISSING: {OUT}"); return 1
    cur = open(OUT).read()
    if cur == text:
        print(f"OK: {OUT} matches a fresh extraction from the pinned upstream file")
        return 0
    print(f"MISMATCH: {OUT} differs from a fresh extraction — re-run with --write "
          "and review the diff")
    return 1


if __name__ == "__main__":
    sys.exit(main())