crisis/tests/test_live_agent.py

"""Tests for LiveClaudeAgent — uses a fake Anthropic client (no real API calls)."""

from dataclasses import dataclass
from typing import Any

import pytest

from crisis_agents.claim import Claim
from crisis_agents.live_agent import LiveClaudeAgent


# ---------------------------------------------------------------------------
# Fakes — we never hit the real Anthropic API in CI.
# ---------------------------------------------------------------------------

@dataclass
class _FakeContentBlock:
    type: str
    text: str


@dataclass
class _FakeResponse:
    content: list[_FakeContentBlock]


class _FakeAnthropicClient:
    """Stand-in for anthropic.Anthropic that returns whatever JSON we hand it."""

    def __init__(self, scripted_responses: list[str]):
        self._responses = list(scripted_responses)
        self.calls: list[dict[str, Any]] = []

        # The real SDK exposes .messages.create; mirror that.
        outer = self

        class _MessagesProxy:
            def create(self_inner, **kwargs):
                outer.calls.append(kwargs)
                text = outer._responses.pop(0) if outer._responses else "[]"
                return _FakeResponse(content=[_FakeContentBlock("text", text)])

        self.messages = _MessagesProxy()


# ---------------------------------------------------------------------------
# The statements + reference doc fixture
# ---------------------------------------------------------------------------

_STATEMENTS = [
    {"id": "s01", "text": "Water boils at 100C at standard pressure."},
    {"id": "s02", "text": "Pluto is still classified as a planet by the IAU."},
]
_REF = "Water boils at 100C. Pluto was reclassified to a dwarf planet in 2006."


class TestLiveClaudeAgent:

    def test_parses_clean_json_response(self):
        response = (
            '[{"statement_id":"s01","verdict":"true","confidence":0.95,"evidence":"per ref"},'
            ' {"statement_id":"s02","verdict":"false","confidence":0.9,"evidence":"per ref"}]'
        )
        client = _FakeAnthropicClient([response])
        agent = LiveClaudeAgent(
            "agent_alpha", reference_doc=_REF,
            statements=_STATEMENTS, client=client,
        )
        turns = agent.next_turn(turn=0, received_claims=[])
        assert len(turns) == 2
        assert {t.claim.statement_id for t in turns} == {"s01", "s02"}
        verdicts = {t.claim.statement_id: t.claim.verdict for t in turns}
        assert verdicts == {"s01": "true", "s02": "false"}

    def test_strips_markdown_fences(self):
        """Claude sometimes wraps JSON in ```json fences despite instructions."""
        response = (
            "```json\n"
            '[{"statement_id":"s01","verdict":"true","confidence":0.9,"evidence":"ok"}]\n'
            "```\n"
        )
        client = _FakeAnthropicClient([response])
        agent = LiveClaudeAgent(
            "agent_alpha", reference_doc=_REF,
            statements=_STATEMENTS, client=client,
        )
        turns = agent.next_turn(turn=0, received_claims=[])
        assert len(turns) == 1
        assert turns[0].claim.statement_id == "s01"

    def test_returns_empty_on_malformed_response(self):
        client = _FakeAnthropicClient(["not json at all"])
        agent = LiveClaudeAgent(
            "agent_alpha", reference_doc=_REF,
            statements=_STATEMENTS, client=client,
        )
        turns = agent.next_turn(turn=0, received_claims=[])
        assert turns == []

    def test_skips_invalid_claim_objects_in_response(self):
        response = (
            '[{"statement_id":"s01","verdict":"true","confidence":0.9,"evidence":"ok"},'
            ' "not a dict",'
            ' {"statement_id":"s02","verdict":"bogus","confidence":0.5,"evidence":"x"}]'
        )
        client = _FakeAnthropicClient([response])
        agent = LiveClaudeAgent(
            "agent_alpha", reference_doc=_REF,
            statements=_STATEMENTS, client=client,
        )
        turns = agent.next_turn(turn=0, received_claims=[])
        # Only the first item passes validation: bogus verdict and non-dict get skipped.
        assert len(turns) == 1
        assert turns[0].claim.statement_id == "s01"

    def test_already_adjudicated_statements_are_skipped(self):
        response_1 = '[{"statement_id":"s01","verdict":"true","confidence":0.9,"evidence":"ok"}]'
        response_2 = '[{"statement_id":"s02","verdict":"false","confidence":0.9,"evidence":"ok"}]'
        client = _FakeAnthropicClient([response_1, response_2])
        agent = LiveClaudeAgent(
            "agent_alpha", reference_doc=_REF,
            statements=_STATEMENTS, client=client,
        )
        # First call adjudicates s01
        first = agent.next_turn(turn=0, received_claims=[])
        assert {t.claim.statement_id for t in first} == {"s01"}

        # Second call should only ask about s02 (s01 is already done)
        second = agent.next_turn(turn=1, received_claims=[])
        assert {t.claim.statement_id for t in second} == {"s02"}

        # The prompt sent for the second call should NOT mention s01
        second_call = client.calls[1]
        user_msg = second_call["messages"][0]["content"]
        assert "s02:" in user_msg
        # s01 was previously adjudicated; it should not appear in the
        # "STATEMENTS TO ADJUDICATE" block of the second prompt.
        statements_section = user_msg.split("=== STATEMENTS TO ADJUDICATE ===")[1]
        next_section_start = statements_section.find("===")
        statements_only = statements_section[:next_section_start]
        assert "s01:" not in statements_only

    def test_evidence_length_is_truncated(self):
        long_evidence = "x" * 500
        response = (
            f'[{{"statement_id":"s01","verdict":"true","confidence":0.9,'
            f'"evidence":"{long_evidence}"}}]'
        )
        client = _FakeAnthropicClient([response])
        agent = LiveClaudeAgent(
            "agent_alpha", reference_doc=_REF,
            statements=_STATEMENTS, client=client,
        )
        turns = agent.next_turn(turn=0, received_claims=[])
        assert len(turns) == 1
        assert len(turns[0].claim.evidence) == Claim.EVIDENCE_MAX_LEN