mirror of
https://github.com/saymrwulf/ltl-accumulator-verified.git
synced 2026-09-04 20:03:44 +00:00
mrwulf
8530cc73f4
This repository has the estate's strongest gates, which makes them the most valuable to switch off. Until now every one of them was executed by scripts that nothing pinned. Phase 0c requires every harness file to match HARNESS.sha256 — 15 files: check.sh, lean-guard, inventory_gate.sh, run_bare.sh, all three self-tests, both audit drivers (Proofs/Inventory.lean, Proofs/AxiomCheck.lean), the policy tables (inventory-allowlist.txt, AUDIT-MANIFEST.txt), the toolchain pin, the fidelity harness and its Python transcription, and the extracted model. WHICH files must be pinned is policy and lives in check.sh, never in the map being consulted: the required set is derived from the filesystem (the executable bit, plus gen/**.lean, plus an explicit list for the rest), so deleting a pin entry is a set mismatch rather than a silent un-pinning. gen/LTLAcc/HashExternal.lean was previously bound by nothing at all — it was compiled and trusted. It is now pinned, and the derivation is by set, so a new model file fails closed. selftest_audit.sh case 9 is split rather than relabelled. Phase 0c now catches an unpinned rogue gen module BEFORE the dead-file gate runs, so asserting only the new diagnostic would have quietly retired the dead-file gate from the test suite. 9a asserts the harness-set mismatch on the unpinned file; 9b pins it — an author who added it deliberately — and asserts Phase 2 still dies with DEAD FILE (gen). Ten cases now, all defeated. KNOWN-GAPS and the trusted base record the circularity plainly: an author who edits a script and refreshes its pin in one commit passes every phase. The pin removes the silent path, not the possibility. Review at the pinned commit remains the consumer's protection. Verified green after the fix: button (75s), harness self-test, binding self-test, and the ten-case audit self-test. ATTESTATION GREEN (Lean + fidelity), all fidelity case counts identical to the pre-change run. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
153 lines
7.7 KiB
Bash
Executable file
153 lines
7.7 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# selftest_audit.sh — adversarial self-test of the audit gate.
|
|
#
|
|
# Review round 2 (GPT H1 / Claude NEW-1) demonstrated that the old
|
|
# source-regex coverage gate was evadable. This script proves the
|
|
# replacement is not, by ATTACKING THE SHIPPING LOGIC: every case copies
|
|
# the corpus to a scratch tree, injects a declaration the old gate could
|
|
# not see, recompiles through lean-guard, and asserts that the exact
|
|
# inventory_gate.sh used by check.sh Phase 3b FAILS with the expected
|
|
# diagnosis. The two unmanifested-module cases run the full check.sh.
|
|
#
|
|
# Cases (release condition 2 of the round-2 GPT review, plus two):
|
|
# 0 positive control: pristine tree must PASS (guards against a gate
|
|
# that fails everything — a vacuous self-test)
|
|
# 1 attributed theorem @[simp] theorem …
|
|
# 2 indented theorem (leading whitespace)
|
|
# 3 private theorem private theorem …
|
|
# 4 instance declaration instance … : Nonempty Nat
|
|
# 5 nested namespace collision LTLAcc.Hidden.MTH vs audited LTLAcc.MTH
|
|
# 6 smuggled axiom axiom rogue : True
|
|
# 7 deleted declaration STALE direction of the diff
|
|
# 8 unmanifested Proofs/ module full check.sh must die: DEAD FILE
|
|
# 9 unmanifested gen/ module full check.sh must die: DEAD FILE (gen)
|
|
#
|
|
# Run AFTER a green check.sh (needs compiled .oleans in the tree).
|
|
# All Lean work goes through lean-guard (memory-capped, single-flight).
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
set -euo pipefail
|
|
AENEAS_ENV="${AENEAS_ENV:-$HOME/aeneas-toolchain/env.sh}"
|
|
[ -f "$AENEAS_ENV" ] || { echo "FATAL: Aeneas environment not found: $AENEAS_ENV"; exit 1; }
|
|
source "$AENEAS_ENV"
|
|
SRC="$(cd "$(dirname "$0")" && pwd)"
|
|
AENEAS_LEAN="$AENEAS_HOME/backends/lean"
|
|
CORES="${LEAN_MAX_CORES:-0-3}"
|
|
|
|
WORK=$(mktemp -d /tmp/acc-selftest-XXXX)
|
|
trap 'echo "(scratch tree kept for inspection: $WORK)"' ERR
|
|
echo "=== audit-gate self-test (scratch: $WORK) ==="
|
|
cp -a "$SRC" "$WORK/verification"
|
|
T="$WORK/verification"
|
|
cp "$T/Proofs/PinStore.lean" "$T/PinStore.pristine"
|
|
|
|
# Recompile the injected leaf module + the inventory, then run the gate.
|
|
# Returns the gate's exit code; gate output goes to $T/gate.out.
|
|
run_gate() {
|
|
cd "$AENEAS_LEAN"
|
|
lake env bash -c "
|
|
set -euo pipefail
|
|
cd '$T' && export LEAN_PATH=\"\$LEAN_PATH:$T/gen:$T\"
|
|
LEAN_TIMEOUT=600 LEAN_MAX_CORES=$CORES '$T/lean-guard' Proofs/PinStore.lean >/dev/null 2>&1
|
|
LEAN_TIMEOUT=600 LEAN_MAX_CORES=$CORES '$T/lean-guard' Proofs/Inventory.lean
|
|
" > "$T/inv.out" 2>&1 || { echo " (inventory compile failed — see $T/inv.out)"; return 99; }
|
|
"$T/inventory_gate.sh" "$T/inv.out" "$T/inventory-allowlist.txt" > "$T/gate.out" 2>&1
|
|
}
|
|
|
|
expect_fail() { # $1=case label $2=grep pattern expected in gate output
|
|
local rc=0; run_gate || rc=$?
|
|
if [ "$rc" = 0 ]; then
|
|
echo " ✗ $1: gate PASSED but must fail"; echo "SELF-TEST FAILED"; exit 1
|
|
elif [ "$rc" = 99 ]; then
|
|
echo " ✗ $1: injected code did not compile (case is vacuous)"; exit 1
|
|
elif ! grep -q "$2" "$T/gate.out"; then
|
|
echo " ✗ $1: gate failed but without expected diagnosis '$2':"
|
|
sed 's/^/ /' "$T/gate.out"; exit 1
|
|
fi
|
|
echo " ✓ $1: gate fails with $(grep -c "$2" "$T/gate.out") '$2' line(s)"
|
|
}
|
|
|
|
restore() { cp "$T/PinStore.pristine" "$T/Proofs/PinStore.lean"; }
|
|
|
|
# 0 — positive control
|
|
if run_gate; then echo " ✓ case 0 control: pristine tree passes the gate"; else
|
|
echo " ✗ case 0 control: pristine tree FAILED the gate:"; sed 's/^/ /' "$T/gate.out"; exit 1; fi
|
|
|
|
# 1 — attributed
|
|
restore; printf '\n@[simp] theorem smuggled_attr : 1 = 1 := rfl\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 1 attributed theorem" "UNCLASSIFIED.*smuggled_attr"
|
|
|
|
# 2 — indented
|
|
restore; printf '\n theorem smuggled_indent : 3 = 3 := rfl\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 2 indented theorem" "UNCLASSIFIED.*smuggled_indent"
|
|
|
|
# 3 — private
|
|
restore; printf '\nprivate theorem smuggled_private : 2 = 2 := rfl\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 3 private theorem" "UNCLASSIFIED.*_private.*smuggled_private"
|
|
|
|
# 4 — instance
|
|
restore; printf '\ninstance smuggledInst : Nonempty Nat := ⟨0⟩\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 4 instance" "UNCLASSIFIED.*smuggledInst"
|
|
|
|
# 5 — nested namespace reusing an audited basename
|
|
restore; printf '\nnamespace LTLAcc.Hidden\ntheorem MTH : 1 = 1 := rfl\nend LTLAcc.Hidden\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 5 namespace collision (LTLAcc.Hidden.MTH)" "UNCLASSIFIED.*LTLAcc\.Hidden\.MTH"
|
|
|
|
# 6 — smuggled axiom
|
|
restore; printf '\naxiom rogue : True\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 6 smuggled axiom" "AXIOM SURFACE DRIFT"
|
|
|
|
# 7 — deleted declaration (STALE direction)
|
|
restore
|
|
python3 - "$T/Proofs/PinStore.lean" <<'EOF'
|
|
import sys
|
|
p = sys.argv[1]; s = open(p).read()
|
|
# drop the trailing nonvacuity guard (a leaf theorem nothing imports),
|
|
# including its doc comment — an orphaned /-- ... -/ would not compile
|
|
i = s.rindex("/-- Permanent non-vacuity witness for pin_prefix_correct")
|
|
j = s.index("end LTLAcc", i)
|
|
open(p, "w").write(s[:i] + s[j:])
|
|
EOF
|
|
expect_fail "case 7 deleted declaration" "STALE.*pin_prefix_nonvacuous"
|
|
|
|
restore
|
|
rm -f "$T/PinStore.pristine"
|
|
|
|
# 8 — unmanifested Proofs/ module (full check.sh; dies in Phase 2)
|
|
printf '/- rogue -/\ntheorem rogue_thm : 1 = 1 := rfl\n' > "$T/Proofs/Rogue.lean"
|
|
if SKIP_FIDELITY=1 "$T/check.sh" > "$T/check8.out" 2>&1; then
|
|
echo " ✗ case 8: check.sh PASSED with unmanifested Proofs/Rogue.lean"; exit 1
|
|
fi
|
|
grep -q "DEAD FILE: Proofs/Rogue.lean" "$T/check8.out" || {
|
|
echo " ✗ case 8: check.sh failed without DEAD FILE diagnosis"; tail -5 "$T/check8.out"; exit 1; }
|
|
echo " ✓ case 8 unmanifested Proofs module: check.sh dies with DEAD FILE"
|
|
rm -f "$T/Proofs/Rogue.lean"
|
|
|
|
# 9 — unmanifested gen/ module. TWO gates stand here now and the case
|
|
# exercises BOTH, because asserting only the outer one would quietly
|
|
# retire the inner one from the test suite.
|
|
#
|
|
# 9a: Phase 0c (added 2026-07-29) derives the required pin set from
|
|
# gen/**.lean, so an unpinned model file is caught before compilation.
|
|
# 9b: with the rogue file pinned — i.e. an author who added it
|
|
# deliberately — the dead-file gate in Phase 2 must still catch that it is
|
|
# absent from the compile manifest.
|
|
printf '/- rogue -/\ntheorem rogue_gen : 1 = 1 := rfl\n' > "$T/gen/LTLAcc/Rogue.lean"
|
|
if SKIP_FIDELITY=1 "$T/check.sh" > "$T/check9a.out" 2>&1; then
|
|
echo " ✗ case 9a: check.sh PASSED with an unpinned gen/LTLAcc/Rogue.lean"; exit 1
|
|
fi
|
|
grep -q "does not match HARNESS.sha256" "$T/check9a.out" || {
|
|
echo " ✗ case 9a: check.sh failed without the harness-set diagnosis"; tail -5 "$T/check9a.out"; exit 1; }
|
|
echo " ✓ case 9a unpinned gen module: Phase 0c dies with a harness-set mismatch"
|
|
|
|
( cd "$T" && sha256sum gen/LTLAcc/Rogue.lean >> HARNESS.sha256 )
|
|
if SKIP_FIDELITY=1 "$T/check.sh" > "$T/check9b.out" 2>&1; then
|
|
echo " ✗ case 9b: check.sh PASSED with unmanifested gen/LTLAcc/Rogue.lean"; exit 1
|
|
fi
|
|
grep -q "DEAD FILE (gen): gen/LTLAcc/Rogue.lean" "$T/check9b.out" || {
|
|
echo " ✗ case 9b: check.sh failed without DEAD FILE (gen) diagnosis"; tail -5 "$T/check9b.out"; exit 1; }
|
|
echo " ✓ case 9b pinned but unmanifested: check.sh dies with DEAD FILE (gen)"
|
|
|
|
rm -rf "$WORK"
|
|
trap - ERR
|
|
echo "=== SELF-TEST GREEN: 10 attack cases defeated + positive control ==="
|