Harden teaching layer, add notebook execution tests, fix repo hygiene

Quality fixes:
- Add deprecation warnings to 5 silent no-op legacy wrappers in assess.py
- Remove dead code in tracker.py score_by_section (unused first loop)
- Remove unused variable in assess.py _check_order
- Fix .gitignore: add progress JSONs, checkpoints, .coverage, .DS_Store, LaTeX aux
- Fix "all three plans" → "all four plans" in learning_objectives.md
- Add teaching/ package to README project tree
- Add compendium to README paper tree

Testing:
- Add 43 unit tests for teaching/assess.py and tracker.py (quiz, predict_choice,
  reflect, order, checkpoint_summary, legacy wrapper deprecation warnings,
  tracker scoring, persistence, mastery calculation)
- Add notebook execution test suite (nbclient): all 11 notebooks execute without
  errors in a fresh kernel, structural validation (valid JSON, has code cells,
  has assessments, section parameters, learning objectives document)
- Overall test count: 185 passing (was 107), coverage: 85% (was ~25% in tests)

Toolchain:
- Add pytest-cov, ruff, nbclient, nbformat to dev dependencies
- Add ruff config (E, F, W, I, UP, B, SIM rules)
- Add coverage config with term-missing output
- Fix all ruff lint issues across src/ and tests/ (import sorting, unused imports)
- Fix Plan D notebook paths (configs/rungs → ../../configs/rungs)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
saymrwulf 2026-04-15 15:34:37 +02:00
parent 5f8b584210
commit 5d0c28f939
27 changed files with 701 additions and 78 deletions

27
.gitignore vendored
View file

@ -1,6 +1,33 @@
# Python
.venv/
__pycache__/
.pytest_cache/
*.pyc
*.egg-info/
# Coverage
.coverage
htmlcov/
# OS
.DS_Store
# Data / runtime artifacts
data/
# Notebook checkpoints and progress
.ipynb_checkpoints/
*_progress.json
*_executed.ipynb
# LaTeX auxiliary files
paper/*.aux
paper/*.log
paper/*.out
paper/*.toc
paper/*.fls
paper/*.fdb_latexmk
paper/*.synctex.gz
# Ruff
.ruff_cache/

View file

@ -46,12 +46,17 @@ autoresearch-quantum/
│ │ └── runner.py AutoresearchHarness orchestrator
│ ├── scoring/
│ │ └── score.py WAC + factory throughput scorers
│ └── search/
│ ├── challengers.py Neighbour generation with dedup
│ └── strategies.py NeighborWalk, RandomCombo, LessonGuided
│ ├── search/
│ │ ├── challengers.py Neighbour generation with dedup
│ │ └── strategies.py NeighborWalk, RandomCombo, LessonGuided
│ └── teaching/
│ ├── assess.py Widget-based quizzes, predictions, reflections
│ └── tracker.py LearningTracker — per-student progress tracking
├── paper/
│ ├── autoresearch_quantum.tex Full technical paper (LaTeX)
│ └── autoresearch_quantum.pdf Compiled PDF (19 pages)
│ ├── autoresearch_quantum.pdf Compiled PDF (19 pages)
│ ├── compendium.tex Companion textbook (LaTeX)
│ └── compendium.pdf Compiled PDF (36 pages)
├── notebooks/
│ ├── plan_a/ Bottom-up: 3 sequential notebooks
│ │ ├── 01_encoded_magic_state.ipynb

View file

@ -1,7 +1,7 @@
# Learning Objectives — Per Notebook, Per Section
Each objective has a Bloom level and a matched assessment type.
All three plans teach the same core material; the pedagogical approach differs.
All four plans teach the same core material; the pedagogical approach differs.
---

View file

@ -297,7 +297,7 @@
"source": [
"from autoresearch_quantum.config import load_rung_config\n",
"\n",
"rung_config = load_rung_config(\"configs/rungs/rung1.yaml\")\n",
"rung_config = load_rung_config(\"../../configs/rungs/rung1.yaml\")\n",
"sweep_results = {}\n",
"\n",
"for opt in [1, 2, 3]:\n",

View file

@ -113,7 +113,7 @@
"cell_type": "code",
"metadata": {},
"source": [
"rung_config = load_rung_config(\"configs/rungs/rung1.yaml\")\n",
"rung_config = load_rung_config(\"../../configs/rungs/rung1.yaml\")\n",
"incumbent_spec = rung_config.bootstrap_incumbent\n",
"print(\"Bootstrap incumbent (the starting point):\")\n",
"for field in [\"seed_style\", \"encoder_style\", \"verification\",\n",

View file

@ -29,6 +29,10 @@ notebooks = [
]
dev = [
"pytest>=9,<10",
"pytest-cov>=6,<7",
"ruff>=0.11,<1",
"nbclient>=0.10,<1",
"nbformat>=5,<6",
]
[project.scripts]
@ -43,3 +47,25 @@ where = ["src"]
[tool.pytest.ini_options]
pythonpath = ["src"]
testpaths = ["tests"]
addopts = "--cov=autoresearch_quantum --cov-report=term-missing --cov-config=pyproject.toml"
[tool.coverage.run]
source = ["autoresearch_quantum"]
[tool.coverage.report]
show_missing = true
skip_empty = true
exclude_lines = [
"pragma: no cover",
"if TYPE_CHECKING:",
"if __name__ == .__main__.",
]
[tool.ruff]
target-version = "py311"
line-length = 120
src = ["src", "tests"]
[tool.ruff.lint]
select = ["E", "F", "W", "I", "UP", "B", "SIM"]
ignore = ["E501"]

View file

@ -657,7 +657,7 @@ justified."""))
cells.append(code("""\
from autoresearch_quantum.config import load_rung_config
rung_config = load_rung_config("configs/rungs/rung1.yaml")
rung_config = load_rung_config("../../configs/rungs/rung1.yaml")
sweep_results = {}
for opt in [1, 2, 3]:
@ -842,7 +842,7 @@ The ratchet works like this:
6. Repeat until patience runs out."""))
cells.append(code("""\
rung_config = load_rung_config("configs/rungs/rung1.yaml")
rung_config = load_rung_config("../../configs/rungs/rung1.yaml")
incumbent_spec = rung_config.bootstrap_incumbent
print("Bootstrap incumbent (the starting point):")
for field in ["seed_style", "encoder_style", "verification",

View file

@ -1,5 +1,4 @@
from .cli import main
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -5,7 +5,6 @@ from math import pi
from qiskit import QuantumCircuit
from qiskit.quantum_info import SparsePauliOp, Statevector
DATA_QUBITS = 4
MAGIC_PREP_QUBIT = 0
SPECTATOR_LOGICAL_QUBIT = 1

View file

@ -1,7 +1,8 @@
from __future__ import annotations
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Mapping
from typing import Any
import yaml

View file

@ -1,9 +1,10 @@
from __future__ import annotations
from collections import Counter
from collections.abc import Iterable
from math import sqrt
from statistics import fmean, pstdev
from typing import Any, Iterable
from typing import Any
def local_memory_records(memory: list[str], creg_names: list[str]) -> list[dict[str, str]]:

View file

@ -1,7 +1,6 @@
from __future__ import annotations
import logging
from dataclasses import replace
from statistics import fmean, stdev
from ..models import ExperimentSpec, RungConfig, TransferReport

View file

@ -2,7 +2,7 @@ from __future__ import annotations
from collections import defaultdict
from itertools import combinations
from statistics import fmean, stdev
from statistics import fmean
from typing import Any
from ..models import LessonFeedback, SearchRule, SearchSpaceConfig

View file

@ -1,13 +1,13 @@
from __future__ import annotations
from dataclasses import asdict, dataclass, field, replace
from datetime import datetime, timezone
from datetime import UTC, datetime
from hashlib import sha1
from typing import Any
def utc_timestamp() -> str:
return datetime.now(timezone.utc).isoformat()
return datetime.now(UTC).isoformat()
def short_hash(payload: str, length: int = 10) -> str:
@ -33,7 +33,7 @@ class ExperimentSpec:
repeats: int = 3
notes: str = ""
def with_updates(self, **changes: Any) -> "ExperimentSpec":
def with_updates(self, **changes: Any) -> ExperimentSpec:
if "initial_layout" in changes and isinstance(changes["initial_layout"], list):
changes["initial_layout"] = tuple(changes["initial_layout"])
return replace(self, **changes)

View file

@ -1,8 +1,7 @@
from __future__ import annotations
import json
import logging
from dataclasses import asdict, replace
from dataclasses import replace
from typing import Any
from ..execution.local import LocalCheapExecutor
@ -19,7 +18,7 @@ from ..models import (
generate_experiment_id,
)
from ..persistence.store import ResearchStore
from ..search.challengers import GeneratedChallenger, mutation_summary
from ..search.challengers import mutation_summary
from ..search.strategies import CompositeGenerator, default_composite
logger = logging.getLogger(__name__)

View file

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Callable
from collections.abc import Callable
from ..models import EvaluationMetrics, FactoryMetrics, QualityWeights, ScoreConfig

View file

@ -1,21 +1,21 @@
"""Teaching toolkit — widget-based active learning and progress tracking."""
from autoresearch_quantum.teaching.tracker import LearningTracker
from autoresearch_quantum.teaching.assess import (
quiz,
predict_choice,
reflect,
order,
checkpoint_summary,
# Legacy aliases for backwards compatibility
multiple_choice,
predict,
check_prediction,
numerical_answer,
free_response,
checkpoint_summary,
code_challenge,
concept_sort,
free_response,
# Legacy aliases for backwards compatibility
multiple_choice,
numerical_answer,
order,
predict,
predict_choice,
quiz,
reflect,
)
from autoresearch_quantum.teaching.tracker import LearningTracker
__all__ = [
"LearningTracker",

View file

@ -1,7 +1,8 @@
"""Assessment primitives — widget-based, click-to-answer, visually distinct from code."""
from __future__ import annotations
from typing import Any, Sequence
import warnings
from typing import Any
import ipywidgets as widgets
from IPython.display import HTML, display
@ -314,7 +315,6 @@ def order(
tie_map[item] = gi
# For each position, check the student's item is valid
# Items in the same tie group can swap positions with each other
used = list(correct_order)
for i, s_item in enumerate(student):
c_item = correct_order[i]
if s_item == c_item:
@ -413,29 +413,49 @@ def multiple_choice(tracker, qid, question, options, correct, answer="?",
quiz(tracker, qid, question, opt_list, correct_idx, bloom, explanation)
def predict(tracker, qid, question, your_prediction="?", bloom="understand"):
"""Legacy wrapper — now a no-op since predict_choice replaces the pair."""
pass
"""Legacy wrapper — use predict_choice() instead."""
warnings.warn(
"predict() is deprecated and does nothing. Use predict_choice() instead.",
DeprecationWarning, stacklevel=2,
)
def check_prediction(tracker, qid, actual_value=None, was_correct=False, explanation=""):
"""Legacy wrapper — no-op."""
pass
"""Legacy wrapper — use predict_choice() instead."""
warnings.warn(
"check_prediction() is deprecated and does nothing. Use predict_choice() instead.",
DeprecationWarning, stacklevel=2,
)
def numerical_answer(tracker, qid, question, answer=0.0, correct=0.0,
tolerance=0.01, bloom="apply", explanation=""):
"""Legacy wrapper — converts to MCQ with the correct value and distractors."""
pass
"""Legacy wrapper — use quiz() instead."""
warnings.warn(
"numerical_answer() is deprecated and does nothing. Use quiz() instead.",
DeprecationWarning, stacklevel=2,
)
def free_response(tracker, qid, question, answer="?", bloom="evaluate", model_answer=""):
"""Legacy wrapper — redirects to reflect()."""
warnings.warn(
"free_response() is deprecated. Use reflect() directly.",
DeprecationWarning, stacklevel=2,
)
reflect(tracker, qid, question, model_answer, bloom)
def code_challenge(tracker, qid, description, test_passed=False,
bloom="apply", hint="", explanation=""):
"""Legacy wrapper — no-op (code challenges stay as code cells)."""
pass
"""Legacy wrapper — no replacement; use code cells with assertions."""
warnings.warn(
"code_challenge() is deprecated and does nothing. Use code cells with assertions.",
DeprecationWarning, stacklevel=2,
)
def concept_sort(tracker, qid, instruction, student_order=None,
correct_order=None, bloom="analyze", explanation=""):
"""Legacy wrapper — redirects to order()."""
"""Legacy wrapper — use order() instead."""
warnings.warn(
"concept_sort() is deprecated. Use order() directly.",
DeprecationWarning, stacklevel=2,
)
if correct_order:
order(tracker, qid, instruction, list(correct_order), list(correct_order), bloom, explanation)

View file

@ -10,7 +10,6 @@ from typing import Any
from IPython.display import HTML, display
# Bloom's taxonomy levels, ordered by cognitive demand
BLOOM_LEVELS = ["remember", "understand", "apply", "analyze", "evaluate", "create"]
@ -89,35 +88,28 @@ class LearningTracker:
self.attempts.append(attempt)
# ── queries ─────────────────────────────────────────────────────────
def score_by_section(self) -> dict[str, dict[str, int]]:
"""Returns {section: {correct: n, incorrect: n, total: n, pct: float}}."""
sections: dict[str, dict[str, int]] = defaultdict(lambda: {"correct": 0, "incorrect": 0, "total": 0})
seen: set[str] = set()
for a in self.attempts:
if a.correct is None:
continue
# only count the *latest* attempt for each question
key = a.question_id
if key in seen:
# remove previous count
continue
seen.add(key)
# recount with latest attempts only
def score_by_section(self) -> dict[str, dict[str, Any]]:
"""Returns {section: {correct: n, incorrect: n, total: n, pct: float}}.
Only the latest attempt per question is counted.
"""
latest: dict[str, Attempt] = {}
for a in self.attempts:
if a.correct is not None:
latest[a.question_id] = a
sections_out: dict[str, dict[str, Any]] = defaultdict(lambda: {"correct": 0, "incorrect": 0, "total": 0})
sections: dict[str, dict[str, Any]] = defaultdict(
lambda: {"correct": 0, "incorrect": 0, "total": 0},
)
for a in latest.values():
s = sections_out[a.section]
s = sections[a.section]
s["total"] += 1
if a.correct:
s["correct"] += 1
else:
s["incorrect"] += 1
for s in sections_out.values():
for s in sections.values():
s["pct"] = round(100 * s["correct"] / s["total"], 1) if s["total"] else 0.0
return dict(sections_out)
return dict(sections)
def score_by_bloom(self) -> dict[str, dict[str, Any]]:
"""Returns {bloom_level: {correct: n, total: n, pct: float}}."""

View file

@ -14,7 +14,6 @@ from autoresearch_quantum.execution.analysis import (
syndrome_outcomes,
)
# ── local_memory_records ─────────────────────────────────────────────────────
def test_local_memory_records_splits_correctly() -> None:

View file

@ -5,7 +5,6 @@ import pytest
from autoresearch_quantum.cli import _parse_override, build_parser
# ── _parse_override ──────────────────────────────────────────────────────────
def test_parse_override_bool_true() -> None:

View file

@ -1,9 +1,8 @@
"""Tests for codes.four_two_two — all seed and encoder styles."""
from __future__ import annotations
from math import pi
import pytest
from qiskit import QuantumCircuit
from qiskit.quantum_info import Statevector
from autoresearch_quantum.codes.four_two_two import (
@ -11,10 +10,7 @@ from autoresearch_quantum.codes.four_two_two import (
apply_magic_seed,
build_encoder,
build_preparation_circuit,
encoded_magic_statevector,
)
from qiskit import QuantumCircuit
# ── apply_magic_seed ─────────────────────────────────────────────────────────

View file

@ -2,7 +2,6 @@
from __future__ import annotations
from autoresearch_quantum.lessons.feedback import (
build_lesson_feedback,
extract_search_rules,
narrow_search_space,
)

View file

@ -5,9 +5,9 @@ from pathlib import Path
from qiskit.quantum_info import Statevector
from autoresearch_quantum.codes.four_two_two import STABILIZERS, encoded_magic_statevector
from autoresearch_quantum.experiments.encoded_magic_state import build_circuit_bundle
from autoresearch_quantum.execution.local import LocalCheapExecutor
from autoresearch_quantum.execution.transfer import TransferEvaluator
from autoresearch_quantum.experiments.encoded_magic_state import build_circuit_bundle
from autoresearch_quantum.lessons.feedback import (
build_lesson_feedback,
extract_search_rules,
@ -16,7 +16,6 @@ from autoresearch_quantum.lessons.feedback import (
from autoresearch_quantum.models import (
CostWeights,
ExperimentSpec,
FactoryMetrics,
HardwareConfig,
LessonFeedback,
QualityWeights,
@ -30,14 +29,11 @@ from autoresearch_quantum.models import (
)
from autoresearch_quantum.persistence.store import ResearchStore
from autoresearch_quantum.ratchet.runner import AutoresearchHarness
from autoresearch_quantum.scoring.score import factory_throughput_score, score_metrics
from autoresearch_quantum.scoring.score import factory_throughput_score
from autoresearch_quantum.search.challengers import generate_neighbor_challengers
from autoresearch_quantum.search.strategies import (
CompositeGenerator,
LessonGuided,
NeighborWalk,
RandomCombo,
StrategyWeight,
default_composite,
)

126
tests/test_notebooks.py Normal file
View file

@ -0,0 +1,126 @@
"""Notebook execution tests — validates that all notebooks run without errors.
Uses nbclient to execute each notebook in a fresh kernel. This catches:
- Import errors
- Broken code cells
- Widget rendering failures
- API mismatches between notebooks and library code
"""
from __future__ import annotations
import re
import warnings
from pathlib import Path
import nbformat
import pytest
from nbclient import NotebookClient
from nbclient.exceptions import CellExecutionError
NOTEBOOK_DIR = Path("notebooks")
ALL_NOTEBOOKS = sorted(NOTEBOOK_DIR.rglob("*.ipynb"))
# Execution timeout per cell (seconds) — noisy simulation cells can be slow
CELL_TIMEOUT = 180
def _notebook_id(path: Path) -> str:
"""Create a readable test ID from notebook path."""
return str(path.relative_to(NOTEBOOK_DIR)).replace("/", "__").removesuffix(".ipynb")
@pytest.fixture(params=ALL_NOTEBOOKS, ids=[_notebook_id(p) for p in ALL_NOTEBOOKS])
def notebook_path(request):
return request.param
def test_notebook_valid_json(notebook_path):
"""Notebook file is valid JSON and nbformat."""
nb = nbformat.read(str(notebook_path), as_version=4)
assert nb.nbformat == 4
assert len(nb.cells) > 0
def test_notebook_has_code_cells(notebook_path):
"""Every notebook has at least one code cell."""
nb = nbformat.read(str(notebook_path), as_version=4)
code_cells = [c for c in nb.cells if c.cell_type == "code"]
assert len(code_cells) > 0, f"{notebook_path} has no code cells"
def test_notebook_executes(notebook_path):
"""Execute the full notebook in a fresh kernel — no cell may raise."""
nb = nbformat.read(str(notebook_path), as_version=4)
# Normalize to add missing cell IDs (avoids nbformat warnings)
_, nb = nbformat.validator.normalize(nb)
# Set cwd to the notebook's directory so relative paths (e.g. ../../configs)
# resolve correctly, matching how a student would run the notebook.
nb_dir = str(notebook_path.resolve().parent)
client = NotebookClient(
nb,
timeout=CELL_TIMEOUT,
kernel_name="python3",
resources={"metadata": {"path": nb_dir}},
)
try:
client.execute()
except CellExecutionError as exc:
# Extract the failing cell for a clear error message
pytest.fail(
f"Notebook {notebook_path} failed during execution:\n"
f"Cell index: {exc.cell_index if hasattr(exc, 'cell_index') else '?'}\n"
f"{exc}"
)
def test_notebook_quiz_cells_have_section(notebook_path):
"""Every quiz/predict/reflect/order call should specify a section= parameter.
Without a section, tracker scores are all lumped into 'intro', making
checkpoint_summary useless.
"""
nb = nbformat.read(str(notebook_path), as_version=4)
missing_section = []
for i, cell in enumerate(nb.cells):
if cell.cell_type != "code":
continue
src = "".join(cell.source)
for fn in ["quiz(", "predict_choice(", "reflect(", "order("]:
if fn in src and "section=" not in src and "tracker" in src and "LearningTracker" not in src:
missing_section.append((i, fn.rstrip("(")))
if missing_section:
details = ", ".join(f"cell {i} ({fn})" for i, fn in missing_section)
warnings.warn(
f"{notebook_path}: {len(missing_section)} assessment calls missing section= parameter: {details}",
UserWarning,
stacklevel=1,
)
def test_learning_objectives_document_exists():
"""learning_objectives.md exists and covers all four plans."""
obj_path = NOTEBOOK_DIR / "learning_objectives.md"
assert obj_path.exists(), "learning_objectives.md not found"
text = obj_path.read_text()
assert "Plan A" in text
assert "Plan B" in text
assert "Plan C" in text
assert "Plan D" in text
assert "four plans" in text.lower()
def test_every_notebook_has_assessments():
"""Every notebook has at least one assessment cell (quiz/predict/reflect/order)."""
assessment_pattern = re.compile(r"(quiz|predict_choice|reflect|order)\s*\(")
for nb_path in ALL_NOTEBOOKS:
nb = nbformat.read(str(nb_path), as_version=4)
has_assessment = False
for cell in nb.cells:
if cell.cell_type == "code":
src = "".join(cell.source)
if assessment_pattern.search(src) and "LearningTracker" not in src:
has_assessment = True
break
assert has_assessment, f"{nb_path} has no assessment cells"

View file

@ -5,7 +5,6 @@ import pytest
from autoresearch_quantum.models import EvaluationMetrics, QualityWeights, ScoreConfig
from autoresearch_quantum.scoring.score import (
SCORE_REGISTRY,
score_metrics,
weighted_acceptance_cost,
)

441
tests/test_teaching.py Normal file
View file

@ -0,0 +1,441 @@
"""Tests for the teaching layer: assess.py and tracker.py."""
from __future__ import annotations
import json
import warnings
from unittest.mock import MagicMock, patch
import pytest
from autoresearch_quantum.teaching.tracker import LearningTracker
# ============================================================================
# LearningTracker — core logic
# ============================================================================
class TestLearningTracker:
def test_init(self):
t = LearningTracker("test_nb")
assert t.notebook_id == "test_nb"
assert t.attempts == []
assert t.current_section == "intro"
def test_set_section(self):
t = LearningTracker("test_nb")
t.set_section("stabilisers")
assert t.current_section == "stabilisers"
def test_record_appends_attempt(self):
t = LearningTracker("test_nb")
t.set_section("sec1")
t.record("q1", "remember", True, "A", "A")
assert len(t.attempts) == 1
a = t.attempts[0]
assert a.question_id == "q1"
assert a.bloom_level == "remember"
assert a.correct is True
assert a.section == "sec1"
assert a.attempt_number == 1
def test_record_increments_attempt_number(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", False, "B", "A")
t.record("q1", "remember", True, "A", "A")
assert t.attempts[0].attempt_number == 1
assert t.attempts[1].attempt_number == 2
def test_record_none_correct_for_reflections(self):
t = LearningTracker("test_nb")
t.record("r1", "evaluate", None, "my thoughts", "model answer")
assert t.attempts[0].correct is None
class TestScoreBySection:
def test_empty_tracker(self):
t = LearningTracker("test_nb")
assert t.score_by_section() == {}
def test_single_correct(self):
t = LearningTracker("test_nb")
t.set_section("sec1")
t.record("q1", "remember", True, "A", "A")
result = t.score_by_section()
assert result["sec1"]["correct"] == 1
assert result["sec1"]["total"] == 1
assert result["sec1"]["pct"] == 100.0
def test_mixed_results(self):
t = LearningTracker("test_nb")
t.set_section("sec1")
t.record("q1", "remember", True, "A", "A")
t.record("q2", "understand", False, "B", "A")
result = t.score_by_section()
assert result["sec1"]["correct"] == 1
assert result["sec1"]["incorrect"] == 1
assert result["sec1"]["total"] == 2
assert result["sec1"]["pct"] == 50.0
def test_latest_attempt_wins(self):
"""Only the latest attempt per question should count."""
t = LearningTracker("test_nb")
t.set_section("sec1")
t.record("q1", "remember", False, "B", "A")
t.record("q1", "remember", True, "A", "A")
result = t.score_by_section()
assert result["sec1"]["correct"] == 1
assert result["sec1"]["incorrect"] == 0
assert result["sec1"]["total"] == 1
def test_none_correct_excluded(self):
"""Reflections (correct=None) should not appear in section scores."""
t = LearningTracker("test_nb")
t.set_section("sec1")
t.record("r1", "evaluate", None, "my thoughts", "model")
assert t.score_by_section() == {}
def test_multiple_sections(self):
t = LearningTracker("test_nb")
t.set_section("sec1")
t.record("q1", "remember", True, "A", "A")
t.set_section("sec2")
t.record("q2", "apply", False, "B", "A")
result = t.score_by_section()
assert "sec1" in result
assert "sec2" in result
assert result["sec1"]["pct"] == 100.0
assert result["sec2"]["pct"] == 0.0
class TestScoreByBloom:
def test_groups_by_bloom(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", True, "A", "A")
t.record("q2", "apply", False, "B", "A")
result = t.score_by_bloom()
assert result["remember"]["correct"] == 1
assert result["apply"]["correct"] == 0
class TestStruggledQuestions:
def test_no_struggles(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", True, "A", "A")
assert t.struggled_questions() == []
def test_wrong_answer_is_struggled(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", False, "B", "A")
assert "q1" in t.struggled_questions()
def test_multiple_attempts_is_struggled(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", False, "B", "A")
t.record("q1", "remember", True, "A", "A")
assert "q1" in t.struggled_questions()
class TestMasteryScore:
def test_empty(self):
t = LearningTracker("test_nb")
assert t.mastery_score() == 0.0
def test_perfect(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", True, "A", "A")
t.record("q2", "understand", True, "B", "B")
assert t.mastery_score() == 100.0
def test_partial(self):
t = LearningTracker("test_nb")
t.record("q1", "remember", True, "A", "A")
t.record("q2", "understand", False, "B", "A")
assert t.mastery_score() == 50.0
class TestSave:
def test_save_to_explicit_path(self, tmp_path):
t = LearningTracker("test_nb")
t.record("q1", "remember", True, "A", "A")
out = t.save(tmp_path / "progress.json")
assert out.exists()
data = json.loads(out.read_text())
assert data["notebook_id"] == "test_nb"
assert data["mastery_score"] == 100.0
assert len(data["attempts"]) == 1
def test_save_to_save_dir(self, tmp_path):
t = LearningTracker("test_nb", save_dir=tmp_path)
t.record("q1", "remember", True, "A", "A")
out = t.save()
assert out == tmp_path / "test_nb_progress.json"
assert out.exists()
def test_save_default_cwd(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
t = LearningTracker("test_nb")
t.record("q1", "remember", True, "A", "A")
out = t.save()
assert out.name == "test_nb_progress.json"
assert out.exists()
# ============================================================================
# assess.py — widget functions
# ============================================================================
# We mock ipywidgets and IPython.display since we're not running in a notebook.
@pytest.fixture
def mock_display():
"""Mock IPython display to capture display calls."""
with patch("autoresearch_quantum.teaching.assess.display") as mock_d:
yield mock_d
@pytest.fixture
def mock_widgets():
"""Mock ipywidgets so we can test widget construction logic."""
with patch("autoresearch_quantum.teaching.assess.widgets") as mock_w:
# Make VBox return a MagicMock that we can track
mock_w.VBox.return_value = MagicMock(name="VBox")
mock_w.HTML.return_value = MagicMock(name="HTML")
mock_w.RadioButtons.return_value = MagicMock(name="RadioButtons", value=None)
mock_w.Button.return_value = MagicMock(name="Button")
mock_w.Textarea.return_value = MagicMock(name="Textarea", value="")
mock_w.Dropdown.return_value = MagicMock(name="Dropdown", value="(select)")
mock_w.Layout.return_value = MagicMock(name="Layout")
yield mock_w
class TestQuiz:
def test_returns_none(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import quiz
t = LearningTracker("test")
result = quiz(t, "q1", "What?", ["A", "B"], 0, section="s1")
assert result is None
def test_displays_exactly_once(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import quiz
t = LearningTracker("test")
quiz(t, "q1", "What?", ["A", "B"], 0)
assert mock_display.call_count == 1
def test_sets_section(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import quiz
t = LearningTracker("test")
quiz(t, "q1", "What?", ["A", "B"], 0, section="mysec")
assert t.current_section == "mysec"
def test_submit_correct_records(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import quiz
t = LearningTracker("test")
mock_btn = mock_widgets.Button.return_value
mock_radio = mock_widgets.RadioButtons.return_value
quiz(t, "q1", "What?", ["A", "B"], 0, bloom="remember")
# Simulate user selecting correct answer and clicking submit
mock_radio.value = "A"
# Get the on_click callback
on_click_call = mock_btn.on_click.call_args
callback = on_click_call[0][0]
callback(None)
assert len(t.attempts) == 1
assert t.attempts[0].correct is True
assert t.attempts[0].student_answer == "A"
def test_submit_incorrect_records(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import quiz
t = LearningTracker("test")
mock_btn = mock_widgets.Button.return_value
mock_radio = mock_widgets.RadioButtons.return_value
quiz(t, "q1", "What?", ["A", "B"], 0, bloom="remember")
mock_radio.value = "B"
callback = mock_btn.on_click.call_args[0][0]
callback(None)
assert t.attempts[0].correct is False
class TestPredictChoice:
def test_returns_none(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import predict_choice
t = LearningTracker("test")
result = predict_choice(t, "q1", "Predict?", ["A", "B"], 0)
assert result is None
def test_displays_exactly_once(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import predict_choice
t = LearningTracker("test")
predict_choice(t, "q1", "Predict?", ["A", "B"], 0)
assert mock_display.call_count == 1
class TestReflect:
def test_returns_none(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import reflect
t = LearningTracker("test")
result = reflect(t, "r1", "What do you think?")
assert result is None
def test_displays_exactly_once(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import reflect
t = LearningTracker("test")
reflect(t, "r1", "What do you think?")
assert mock_display.call_count == 1
class TestOrder:
def test_returns_none(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import order
t = LearningTracker("test")
result = order(t, "o1", "Sort these:", ["A", "B"], ["A", "B"])
assert result is None
def test_displays_exactly_once(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import order
t = LearningTracker("test")
order(t, "o1", "Sort:", ["A", "B"], ["A", "B"])
assert mock_display.call_count == 1
def test_order_check_exact(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import order
t = LearningTracker("test")
# We need to test the _check_order logic. Since it's nested,
# we test via the submit callback.
mock_btn = mock_widgets.Button.return_value
# Create two dropdowns that will return values
dd1 = MagicMock(value="A", disabled=False)
dd2 = MagicMock(value="B", disabled=False)
mock_widgets.Dropdown.side_effect = [dd1, dd2]
order(t, "o1", "Sort:", ["A", "B"], ["A", "B"])
callback = mock_btn.on_click.call_args[0][0]
callback(None)
assert len(t.attempts) == 1
assert t.attempts[0].correct is True
def test_order_check_with_ties(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import order
t = LearningTracker("test")
mock_btn = mock_widgets.Button.return_value
# Student puts Z before X (tied, so should be correct)
dd1 = MagicMock(value="Z", disabled=False)
dd2 = MagicMock(value="X", disabled=False)
dd3 = MagicMock(value="Y", disabled=False)
mock_widgets.Dropdown.side_effect = [dd1, dd2, dd3]
order(t, "o1", "Sort:", ["X", "Z", "Y"], ["X", "Z", "Y"],
ties=[["X", "Z"]])
callback = mock_btn.on_click.call_args[0][0]
callback(None)
assert t.attempts[0].correct is True
def test_order_check_wrong(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import order
t = LearningTracker("test")
mock_btn = mock_widgets.Button.return_value
dd1 = MagicMock(value="Y", disabled=False)
dd2 = MagicMock(value="X", disabled=False)
dd3 = MagicMock(value="Z", disabled=False)
mock_widgets.Dropdown.side_effect = [dd1, dd2, dd3]
order(t, "o1", "Sort:", ["X", "Z", "Y"], ["X", "Z", "Y"])
callback = mock_btn.on_click.call_args[0][0]
callback(None)
assert t.attempts[0].correct is False
class TestCheckpointSummary:
def test_displays_html(self, mock_display):
with patch("autoresearch_quantum.teaching.assess.HTML"):
from autoresearch_quantum.teaching.assess import checkpoint_summary
t = LearningTracker("test")
t.set_section("sec1")
t.record("q1", "remember", True, "A", "A")
checkpoint_summary(t, "sec1")
assert mock_display.call_count == 1
# ============================================================================
# Legacy wrappers — deprecation warnings
# ============================================================================
class TestLegacyWrappers:
def test_predict_warns(self):
from autoresearch_quantum.teaching.assess import predict
t = LearningTracker("test")
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
predict(t, "q1", "What?")
assert len(w) == 1
assert issubclass(w[0].category, DeprecationWarning)
assert "predict_choice" in str(w[0].message)
def test_check_prediction_warns(self):
from autoresearch_quantum.teaching.assess import check_prediction
t = LearningTracker("test")
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
check_prediction(t, "q1")
assert len(w) == 1
assert issubclass(w[0].category, DeprecationWarning)
def test_numerical_answer_warns(self):
from autoresearch_quantum.teaching.assess import numerical_answer
t = LearningTracker("test")
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
numerical_answer(t, "q1", "How many?")
assert len(w) == 1
assert issubclass(w[0].category, DeprecationWarning)
def test_code_challenge_warns(self):
from autoresearch_quantum.teaching.assess import code_challenge
t = LearningTracker("test")
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
code_challenge(t, "q1", "Do X")
assert len(w) == 1
assert issubclass(w[0].category, DeprecationWarning)
def test_free_response_warns_and_redirects(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import free_response
t = LearningTracker("test")
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
free_response(t, "q1", "Explain?", model_answer="Because X")
assert len(w) == 1
assert issubclass(w[0].category, DeprecationWarning)
# Should still display a reflect widget
assert mock_display.call_count == 1
def test_concept_sort_warns_and_redirects(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import concept_sort
t = LearningTracker("test")
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
concept_sort(t, "q1", "Sort:", correct_order=["A", "B"])
assert len(w) == 1
assert issubclass(w[0].category, DeprecationWarning)
assert mock_display.call_count == 1
def test_multiple_choice_redirects(self, mock_display, mock_widgets):
from autoresearch_quantum.teaching.assess import multiple_choice
t = LearningTracker("test")
# multiple_choice is not deprecated (it works), just redirects
multiple_choice(t, "q1", "What?", {"a": "Alpha", "b": "Beta"}, "a")
assert mock_display.call_count == 1