fips205-source/tests/nist_acvp_vectors/extract_sha2_128s.py

#!/usr/bin/env python3
"""Deterministically re-derive tests/nist_acvp_vectors/SLH-DSA-sigVer-FIPS205/
sha2_128s_extracted.json from the official NIST ACVP-Server vector set.

Why this file exists at all: the ACVP vector file vendored in this repository
contains no SLH-DSA-SHA2-128s sigVer group, so the parameter set the associated
verification work is about had no NIST known-answer verification coverage. The
three 128s groups are therefore taken from upstream.

Why this SCRIPT exists: external review (round 7) observed that a hand-made
extraction with a prose provenance note is not auditable — the first version's
note said "only `sk` dropped" while three further fields had in fact been
removed. The transformation is now executable, pinned, and fails closed:

  * the upstream file's sha256 must match SOURCE_SHA256 exactly;
  * exactly EXPECTED_GROUPS groups must match the parameter set, each with
    EXPECTED_TESTS_PER_GROUP tests;
  * exactly one field, `sk`, is removed, and it must be present to be removed;
  * every other key present upstream is carried through untouched, and the
    script asserts that afterwards;
  * output is canonical (sorted keys off, fixed indent, trailing newline).

Usage:
    python3 extract_sha2_128s.py            # verify the committed file matches
    python3 extract_sha2_128s.py --write    # regenerate it

The upstream file is ~30 MB; it is fetched to a temporary path and not vendored.
"""

import argparse, hashlib, json, os, sys, tempfile, urllib.request

SOURCE_URL = (
    "https://raw.githubusercontent.com/usnistgov/ACVP-Server/master/"
    "gen-val/json-files/SLH-DSA-sigVer-FIPS205/internalProjection.json"
)
SOURCE_SHA256 = "a013fc2104f4ed4799d96d51141f65b965969b2cf10646626a021b6d456ce792"
PARAMETER_SET = "SLH-DSA-SHA2-128s"
EXPECTED_GROUPS = 3
EXPECTED_TESTS_PER_GROUP = 14
DROP_FIELDS = frozenset({"sk"})          # the ONLY per-test removal
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                   "SLH-DSA-sigVer-FIPS205", "sha2_128s_extracted.json")

PROVENANCE = {
    "what": f"{PARAMETER_SET} sigVer test groups, extracted verbatim from the "
            "official NIST ACVP-Server vector set.",
    "why": "The vector file vendored upstream in this directory contains NO "
           f"{PARAMETER_SET} sigVer group (only 192s/256f/SHAKE variants), so the "
           "one parameter set the verification work targets had zero NIST "
           "known-answer verification coverage.",
    "source_url": SOURCE_URL,
    "source_sha256": SOURCE_SHA256,
    "retrieved_utc": "2026-07-28",
    "extraction": "Produced by tests/nist_acvp_vectors/extract_sha2_128s.py, which "
                  "verifies the upstream sha256, selects every testGroup whose "
                  f"parameterSet == {PARAMETER_SET}, and removes exactly ONE "
                  "per-test field: `sk` (the private key, not needed to verify a "
                  "signature). Every other per-test field and all group and "
                  "top-level metadata are carried through unchanged; the script "
                  "asserts this and fails if it is not so.",
    "note": "Test DATA only. Expected outcomes are NIST's `testPassed` field; "
            "`reason` records why a negative case must be rejected.",
    "regenerate": "python3 tests/nist_acvp_vectors/extract_sha2_128s.py --write",
}


def fetch() -> dict:
    with tempfile.NamedTemporaryFile(delete=False, suffix=".json") as fh:
        tmp = fh.name
    try:
        urllib.request.urlretrieve(SOURCE_URL, tmp)
        raw = open(tmp, "rb").read()
    finally:
        os.unlink(tmp)
    got = hashlib.sha256(raw).hexdigest()
    if got != SOURCE_SHA256:
        sys.exit(f"FATAL: upstream sha256 {got} != pinned {SOURCE_SHA256}.\n"
                 "The NIST file changed. Review the delta and update the pin "
                 "deliberately; do not regenerate blindly.")
    return json.loads(raw)


def build(full: dict) -> dict:
    groups = []
    for g in full["testGroups"]:
        if g.get("parameterSet") != PARAMETER_SET:
            continue
        tests = []
        for t in g["tests"]:
            for f in DROP_FIELDS:
                if f not in t:
                    sys.exit(f"FATAL: tcId {t.get('tcId')} has no field {f!r} to drop")
            kept = {k: v for k, v in t.items() if k not in DROP_FIELDS}
            # carried-through invariant, asserted rather than asserted-in-prose
            assert set(kept) == set(t) - DROP_FIELDS, "unexpected per-test key change"
            tests.append(kept)
        if len(tests) != EXPECTED_TESTS_PER_GROUP:
            sys.exit(f"FATAL: group {g['tgId']} has {len(tests)} tests, "
                     f"expected {EXPECTED_TESTS_PER_GROUP}")
        ng = {k: v for k, v in g.items() if k != "tests"}
        ng["tests"] = tests
        groups.append(ng)
    if len(groups) != EXPECTED_GROUPS:
        sys.exit(f"FATAL: found {len(groups)} {PARAMETER_SET} groups, expected {EXPECTED_GROUPS}")
    out = {"_provenance": PROVENANCE}
    for k, v in full.items():                    # all top-level metadata, verbatim
        if k != "testGroups":
            out[k] = v
    out["testGroups"] = groups
    return out


def render(obj: dict) -> str:
    return json.dumps(obj, indent=1) + "\n"


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--write", action="store_true", help="regenerate the committed file")
    args = ap.parse_args()
    text = render(build(fetch()))
    if args.write:
        open(OUT, "w").write(text)
        print(f"wrote {OUT} ({len(text)} bytes)")
        return 0
    if not os.path.exists(OUT):
        print(f"MISSING: {OUT}"); return 1
    cur = open(OUT).read()
    if cur == text:
        print(f"OK: {OUT} matches a fresh extraction from the pinned upstream file")
        return 0
    print(f"MISMATCH: {OUT} differs from a fresh extraction — re-run with --write "
          "and review the diff")
    return 1


if __name__ == "__main__":
    sys.exit(main())