mirror of
https://github.com/saymrwulf/ltl-accumulator-verified.git
synced 2026-09-03 19:53:48 +00:00
mrwulf
9972ab4198
Round-2 external reviews (GPT-5.6 + second Claude) converged on the coverage gate being evadable (H1/NEW-1); GPT additionally proved the kit's fidelity target could not run (H2) and the namespace-collision attack that defeats any source-regex fix. This round adopts GPT's required correction in full: - Proofs/Inventory.lean: declaration inventory read from the compiled Lean environment — every constant of every corpus module, fully qualified, unfiltered (compiler auxiliaries and _private mangles pinned too), with kind and axiom cone; own cone walker cross-checked in-process against core collectAxioms (hard error on divergence). - verification/inventory-allowlist.txt: all 218 constants pinned. - inventory_gate.sh: fail-closed diff both directions (UNCLASSIFIED / STALE), INV-COUNT truncation guard, exactly-one-axiom invariant. - check.sh Phase 3b rewritten around the gate + manifest⇔inventory drift checks + CONES⇔inventory cone cross-check (two independent computations must agree). EXCLUDE table gone (sha256/Bytes are ordinary audited entries now). - selftest_audit.sh: 9 adversarial cases against the production gate (attributed/indented/private/instance, namespace collision, smuggled axiom, deleted decl, unmanifested Proofs/ and gen/ modules) + positive control — all defeated (GPT release condition 2). - M1: recursive orphan-olean guard (caught a stray dev artifact on its first run), gen/ dead-file check, corpus-wide single-axiom pin. - L1/NEW-2: acceptIncl_sound drops the redundant hm (derived from hacc.1); cone unchanged. - M2/M3: STATEMENT-MAP counts 230,271/230,016; non-vacuity guard wording narrowed to what the guards actually certify. - README layer table: stale L4/pin-store rows fixed (missed by both round-2 reviewers AND the round-2 revision — found in self-review). - KNOWN-GAPS 12 (audit-gate lineage + residual limits), 13 (round-2 kit target not self-contained); gap 2 count fixed. - RESPONSE-TO-REVIEWERS.md: round-3 disposition of every finding. Kit round 3 additionally ships the complete stdlib-only import closure of pacta.transparency (content-addressed vs pacta 3d81d53), the clean-extraction fidelity transcript (exit 0, 230,271+230,016, zero mismatches), the ATTESTATION GREEN check.sh transcript, and the self-test transcript. The live LTL remains untouched (12 leaves, root bcd15f9d…); attestation stays blocked pending ePrint decision + author review + explicit operator order. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
135 lines
6.6 KiB
Bash
Executable file
135 lines
6.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# selftest_audit.sh — adversarial self-test of the audit gate.
|
|
#
|
|
# Review round 2 (GPT H1 / Claude NEW-1) demonstrated that the old
|
|
# source-regex coverage gate was evadable. This script proves the
|
|
# replacement is not, by ATTACKING THE SHIPPING LOGIC: every case copies
|
|
# the corpus to a scratch tree, injects a declaration the old gate could
|
|
# not see, recompiles through lean-guard, and asserts that the exact
|
|
# inventory_gate.sh used by check.sh Phase 3b FAILS with the expected
|
|
# diagnosis. The two unmanifested-module cases run the full check.sh.
|
|
#
|
|
# Cases (release condition 2 of the round-2 GPT review, plus two):
|
|
# 0 positive control: pristine tree must PASS (guards against a gate
|
|
# that fails everything — a vacuous self-test)
|
|
# 1 attributed theorem @[simp] theorem …
|
|
# 2 indented theorem (leading whitespace)
|
|
# 3 private theorem private theorem …
|
|
# 4 instance declaration instance … : Nonempty Nat
|
|
# 5 nested namespace collision LTLAcc.Hidden.MTH vs audited LTLAcc.MTH
|
|
# 6 smuggled axiom axiom rogue : True
|
|
# 7 deleted declaration STALE direction of the diff
|
|
# 8 unmanifested Proofs/ module full check.sh must die: DEAD FILE
|
|
# 9 unmanifested gen/ module full check.sh must die: DEAD FILE (gen)
|
|
#
|
|
# Run AFTER a green check.sh (needs compiled .oleans in the tree).
|
|
# All Lean work goes through lean-guard (memory-capped, single-flight).
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
set -euo pipefail
|
|
source ~/aeneas-toolchain/env.sh
|
|
SRC="$(cd "$(dirname "$0")" && pwd)"
|
|
AENEAS_LEAN="$AENEAS_HOME/backends/lean"
|
|
CORES="${LEAN_MAX_CORES:-0-3}"
|
|
|
|
WORK=$(mktemp -d /tmp/acc-selftest-XXXX)
|
|
trap 'echo "(scratch tree kept for inspection: $WORK)"' ERR
|
|
echo "=== audit-gate self-test (scratch: $WORK) ==="
|
|
cp -a "$SRC" "$WORK/verification"
|
|
T="$WORK/verification"
|
|
cp "$T/Proofs/PinStore.lean" "$T/PinStore.pristine"
|
|
|
|
# Recompile the injected leaf module + the inventory, then run the gate.
|
|
# Returns the gate's exit code; gate output goes to $T/gate.out.
|
|
run_gate() {
|
|
cd "$AENEAS_LEAN"
|
|
lake env bash -c "
|
|
set -euo pipefail
|
|
cd '$T' && export LEAN_PATH=\"\$LEAN_PATH:$T/gen:$T\"
|
|
LEAN_TIMEOUT=600 LEAN_MAX_CORES=$CORES '$T/lean-guard' Proofs/PinStore.lean >/dev/null 2>&1
|
|
LEAN_TIMEOUT=600 LEAN_MAX_CORES=$CORES '$T/lean-guard' Proofs/Inventory.lean
|
|
" > "$T/inv.out" 2>&1 || { echo " (inventory compile failed — see $T/inv.out)"; return 99; }
|
|
"$T/inventory_gate.sh" "$T/inv.out" "$T/inventory-allowlist.txt" > "$T/gate.out" 2>&1
|
|
}
|
|
|
|
expect_fail() { # $1=case label $2=grep pattern expected in gate output
|
|
local rc=0; run_gate || rc=$?
|
|
if [ "$rc" = 0 ]; then
|
|
echo " ✗ $1: gate PASSED but must fail"; echo "SELF-TEST FAILED"; exit 1
|
|
elif [ "$rc" = 99 ]; then
|
|
echo " ✗ $1: injected code did not compile (case is vacuous)"; exit 1
|
|
elif ! grep -q "$2" "$T/gate.out"; then
|
|
echo " ✗ $1: gate failed but without expected diagnosis '$2':"
|
|
sed 's/^/ /' "$T/gate.out"; exit 1
|
|
fi
|
|
echo " ✓ $1: gate fails with $(grep -c "$2" "$T/gate.out") '$2' line(s)"
|
|
}
|
|
|
|
restore() { cp "$T/PinStore.pristine" "$T/Proofs/PinStore.lean"; }
|
|
|
|
# 0 — positive control
|
|
if run_gate; then echo " ✓ case 0 control: pristine tree passes the gate"; else
|
|
echo " ✗ case 0 control: pristine tree FAILED the gate:"; sed 's/^/ /' "$T/gate.out"; exit 1; fi
|
|
|
|
# 1 — attributed
|
|
restore; printf '\n@[simp] theorem smuggled_attr : 1 = 1 := rfl\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 1 attributed theorem" "UNCLASSIFIED.*smuggled_attr"
|
|
|
|
# 2 — indented
|
|
restore; printf '\n theorem smuggled_indent : 3 = 3 := rfl\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 2 indented theorem" "UNCLASSIFIED.*smuggled_indent"
|
|
|
|
# 3 — private
|
|
restore; printf '\nprivate theorem smuggled_private : 2 = 2 := rfl\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 3 private theorem" "UNCLASSIFIED.*_private.*smuggled_private"
|
|
|
|
# 4 — instance
|
|
restore; printf '\ninstance smuggledInst : Nonempty Nat := ⟨0⟩\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 4 instance" "UNCLASSIFIED.*smuggledInst"
|
|
|
|
# 5 — nested namespace reusing an audited basename
|
|
restore; printf '\nnamespace LTLAcc.Hidden\ntheorem MTH : 1 = 1 := rfl\nend LTLAcc.Hidden\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 5 namespace collision (LTLAcc.Hidden.MTH)" "UNCLASSIFIED.*LTLAcc\.Hidden\.MTH"
|
|
|
|
# 6 — smuggled axiom
|
|
restore; printf '\naxiom rogue : True\n' >> "$T/Proofs/PinStore.lean"
|
|
expect_fail "case 6 smuggled axiom" "AXIOM SURFACE DRIFT"
|
|
|
|
# 7 — deleted declaration (STALE direction)
|
|
restore
|
|
python3 - "$T/Proofs/PinStore.lean" <<'EOF'
|
|
import sys
|
|
p = sys.argv[1]; s = open(p).read()
|
|
# drop the trailing nonvacuity guard (a leaf theorem nothing imports),
|
|
# including its doc comment — an orphaned /-- ... -/ would not compile
|
|
i = s.rindex("/-- Permanent non-vacuity witness for pin_prefix_correct")
|
|
j = s.index("end LTLAcc", i)
|
|
open(p, "w").write(s[:i] + s[j:])
|
|
EOF
|
|
expect_fail "case 7 deleted declaration" "STALE.*pin_prefix_nonvacuous"
|
|
|
|
restore
|
|
rm -f "$T/PinStore.pristine"
|
|
|
|
# 8 — unmanifested Proofs/ module (full check.sh; dies in Phase 2)
|
|
printf '/- rogue -/\ntheorem rogue_thm : 1 = 1 := rfl\n' > "$T/Proofs/Rogue.lean"
|
|
if SKIP_FIDELITY=1 "$T/check.sh" > "$T/check8.out" 2>&1; then
|
|
echo " ✗ case 8: check.sh PASSED with unmanifested Proofs/Rogue.lean"; exit 1
|
|
fi
|
|
grep -q "DEAD FILE: Proofs/Rogue.lean" "$T/check8.out" || {
|
|
echo " ✗ case 8: check.sh failed without DEAD FILE diagnosis"; tail -5 "$T/check8.out"; exit 1; }
|
|
echo " ✓ case 8 unmanifested Proofs module: check.sh dies with DEAD FILE"
|
|
rm -f "$T/Proofs/Rogue.lean"
|
|
|
|
# 9 — unmanifested gen/ module (full check.sh; dies in Phase 2)
|
|
printf '/- rogue -/\ntheorem rogue_gen : 1 = 1 := rfl\n' > "$T/gen/LTLAcc/Rogue.lean"
|
|
if SKIP_FIDELITY=1 "$T/check.sh" > "$T/check9.out" 2>&1; then
|
|
echo " ✗ case 9: check.sh PASSED with unmanifested gen/LTLAcc/Rogue.lean"; exit 1
|
|
fi
|
|
grep -q "DEAD FILE (gen): gen/LTLAcc/Rogue.lean" "$T/check9.out" || {
|
|
echo " ✗ case 9: check.sh failed without DEAD FILE (gen) diagnosis"; tail -5 "$T/check9.out"; exit 1; }
|
|
echo " ✓ case 9 unmanifested gen module: check.sh dies with DEAD FILE (gen)"
|
|
|
|
rm -rf "$WORK"
|
|
trap - ERR
|
|
echo "=== SELF-TEST GREEN: 9 attack cases defeated + positive control ==="
|