From 5d0c28f9398022d05e8c7cf953d6d528d4965a8a Mon Sep 17 00:00:00 2001 From: saymrwulf Date: Wed, 15 Apr 2026 15:34:37 +0200 Subject: [PATCH] Harden teaching layer, add notebook execution tests, fix repo hygiene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quality fixes: - Add deprecation warnings to 5 silent no-op legacy wrappers in assess.py - Remove dead code in tracker.py score_by_section (unused first loop) - Remove unused variable in assess.py _check_order - Fix .gitignore: add progress JSONs, checkpoints, .coverage, .DS_Store, LaTeX aux - Fix "all three plans" → "all four plans" in learning_objectives.md - Add teaching/ package to README project tree - Add compendium to README paper tree Testing: - Add 43 unit tests for teaching/assess.py and tracker.py (quiz, predict_choice, reflect, order, checkpoint_summary, legacy wrapper deprecation warnings, tracker scoring, persistence, mastery calculation) - Add notebook execution test suite (nbclient): all 11 notebooks execute without errors in a fresh kernel, structural validation (valid JSON, has code cells, has assessments, section parameters, learning objectives document) - Overall test count: 185 passing (was 107), coverage: 85% (was ~25% in tests) Toolchain: - Add pytest-cov, ruff, nbclient, nbformat to dev dependencies - Add ruff config (E, F, W, I, UP, B, SIM rules) - Add coverage config with term-missing output - Fix all ruff lint issues across src/ and tests/ (import sorting, unused imports) - Fix Plan D notebook paths (configs/rungs → ../../configs/rungs) Co-Authored-By: Claude Opus 4.6 --- .gitignore | 27 ++ README.md | 13 +- notebooks/learning_objectives.md | 2 +- notebooks/plan_d/experiment_2_noise.ipynb | 2 +- .../plan_d/experiment_3_optimisation.ipynb | 2 +- pyproject.toml | 26 ++ scripts/build_plan_d.py | 4 +- src/autoresearch_quantum/__main__.py | 1 - .../codes/four_two_two.py | 1 - src/autoresearch_quantum/config.py | 3 +- .../execution/analysis.py | 3 +- .../execution/transfer.py | 1 - src/autoresearch_quantum/lessons/feedback.py | 2 +- src/autoresearch_quantum/models.py | 6 +- src/autoresearch_quantum/ratchet/runner.py | 5 +- src/autoresearch_quantum/scoring/score.py | 2 +- src/autoresearch_quantum/teaching/__init__.py | 22 +- src/autoresearch_quantum/teaching/assess.py | 42 +- src/autoresearch_quantum/teaching/tracker.py | 30 +- tests/test_analysis.py | 1 - tests/test_cli.py | 1 - tests/test_codes.py | 6 +- tests/test_feedback.py | 1 - tests/test_harness.py | 8 +- tests/test_notebooks.py | 126 +++++ tests/test_scoring.py | 1 - tests/test_teaching.py | 441 ++++++++++++++++++ 27 files changed, 701 insertions(+), 78 deletions(-) create mode 100644 tests/test_notebooks.py create mode 100644 tests/test_teaching.py diff --git a/.gitignore b/.gitignore index 98905e2..7a7bcd8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,33 @@ +# Python .venv/ __pycache__/ .pytest_cache/ *.pyc *.egg-info/ + +# Coverage +.coverage +htmlcov/ + +# OS +.DS_Store + +# Data / runtime artifacts data/ + +# Notebook checkpoints and progress +.ipynb_checkpoints/ +*_progress.json +*_executed.ipynb + +# LaTeX auxiliary files +paper/*.aux +paper/*.log +paper/*.out +paper/*.toc +paper/*.fls +paper/*.fdb_latexmk +paper/*.synctex.gz + +# Ruff +.ruff_cache/ diff --git a/README.md b/README.md index 71e7459..9fe5c64 100644 --- a/README.md +++ b/README.md @@ -46,12 +46,17 @@ autoresearch-quantum/ │ │ └── runner.py AutoresearchHarness orchestrator │ ├── scoring/ │ │ └── score.py WAC + factory throughput scorers -│ └── search/ -│ ├── challengers.py Neighbour generation with dedup -│ └── strategies.py NeighborWalk, RandomCombo, LessonGuided +│ ├── search/ +│ │ ├── challengers.py Neighbour generation with dedup +│ │ └── strategies.py NeighborWalk, RandomCombo, LessonGuided +│ └── teaching/ +│ ├── assess.py Widget-based quizzes, predictions, reflections +│ └── tracker.py LearningTracker — per-student progress tracking ├── paper/ │ ├── autoresearch_quantum.tex Full technical paper (LaTeX) -│ └── autoresearch_quantum.pdf Compiled PDF (19 pages) +│ ├── autoresearch_quantum.pdf Compiled PDF (19 pages) +│ ├── compendium.tex Companion textbook (LaTeX) +│ └── compendium.pdf Compiled PDF (36 pages) ├── notebooks/ │ ├── plan_a/ Bottom-up: 3 sequential notebooks │ │ ├── 01_encoded_magic_state.ipynb diff --git a/notebooks/learning_objectives.md b/notebooks/learning_objectives.md index bff71f5..c7b3a96 100644 --- a/notebooks/learning_objectives.md +++ b/notebooks/learning_objectives.md @@ -1,7 +1,7 @@ # Learning Objectives — Per Notebook, Per Section Each objective has a Bloom level and a matched assessment type. -All three plans teach the same core material; the pedagogical approach differs. +All four plans teach the same core material; the pedagogical approach differs. --- diff --git a/notebooks/plan_d/experiment_2_noise.ipynb b/notebooks/plan_d/experiment_2_noise.ipynb index 1726cf6..2dcf4a5 100644 --- a/notebooks/plan_d/experiment_2_noise.ipynb +++ b/notebooks/plan_d/experiment_2_noise.ipynb @@ -297,7 +297,7 @@ "source": [ "from autoresearch_quantum.config import load_rung_config\n", "\n", - "rung_config = load_rung_config(\"configs/rungs/rung1.yaml\")\n", + "rung_config = load_rung_config(\"../../configs/rungs/rung1.yaml\")\n", "sweep_results = {}\n", "\n", "for opt in [1, 2, 3]:\n", diff --git a/notebooks/plan_d/experiment_3_optimisation.ipynb b/notebooks/plan_d/experiment_3_optimisation.ipynb index 6185986..5ab6c30 100644 --- a/notebooks/plan_d/experiment_3_optimisation.ipynb +++ b/notebooks/plan_d/experiment_3_optimisation.ipynb @@ -113,7 +113,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "rung_config = load_rung_config(\"configs/rungs/rung1.yaml\")\n", + "rung_config = load_rung_config(\"../../configs/rungs/rung1.yaml\")\n", "incumbent_spec = rung_config.bootstrap_incumbent\n", "print(\"Bootstrap incumbent (the starting point):\")\n", "for field in [\"seed_style\", \"encoder_style\", \"verification\",\n", diff --git a/pyproject.toml b/pyproject.toml index da5d9c2..818ca8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,10 @@ notebooks = [ ] dev = [ "pytest>=9,<10", + "pytest-cov>=6,<7", + "ruff>=0.11,<1", + "nbclient>=0.10,<1", + "nbformat>=5,<6", ] [project.scripts] @@ -43,3 +47,25 @@ where = ["src"] [tool.pytest.ini_options] pythonpath = ["src"] testpaths = ["tests"] +addopts = "--cov=autoresearch_quantum --cov-report=term-missing --cov-config=pyproject.toml" + +[tool.coverage.run] +source = ["autoresearch_quantum"] + +[tool.coverage.report] +show_missing = true +skip_empty = true +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "if __name__ == .__main__.", +] + +[tool.ruff] +target-version = "py311" +line-length = 120 +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM"] +ignore = ["E501"] diff --git a/scripts/build_plan_d.py b/scripts/build_plan_d.py index 687b80c..e07da11 100644 --- a/scripts/build_plan_d.py +++ b/scripts/build_plan_d.py @@ -657,7 +657,7 @@ justified.""")) cells.append(code("""\ from autoresearch_quantum.config import load_rung_config -rung_config = load_rung_config("configs/rungs/rung1.yaml") +rung_config = load_rung_config("../../configs/rungs/rung1.yaml") sweep_results = {} for opt in [1, 2, 3]: @@ -842,7 +842,7 @@ The ratchet works like this: 6. Repeat until patience runs out.""")) cells.append(code("""\ -rung_config = load_rung_config("configs/rungs/rung1.yaml") +rung_config = load_rung_config("../../configs/rungs/rung1.yaml") incumbent_spec = rung_config.bootstrap_incumbent print("Bootstrap incumbent (the starting point):") for field in ["seed_style", "encoder_style", "verification", diff --git a/src/autoresearch_quantum/__main__.py b/src/autoresearch_quantum/__main__.py index a049ad7..bfdcd0c 100644 --- a/src/autoresearch_quantum/__main__.py +++ b/src/autoresearch_quantum/__main__.py @@ -1,5 +1,4 @@ from .cli import main - if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/autoresearch_quantum/codes/four_two_two.py b/src/autoresearch_quantum/codes/four_two_two.py index 3ffa67b..e724c41 100644 --- a/src/autoresearch_quantum/codes/four_two_two.py +++ b/src/autoresearch_quantum/codes/four_two_two.py @@ -5,7 +5,6 @@ from math import pi from qiskit import QuantumCircuit from qiskit.quantum_info import SparsePauliOp, Statevector - DATA_QUBITS = 4 MAGIC_PREP_QUBIT = 0 SPECTATOR_LOGICAL_QUBIT = 1 diff --git a/src/autoresearch_quantum/config.py b/src/autoresearch_quantum/config.py index a07aa1e..3bc0ec3 100644 --- a/src/autoresearch_quantum/config.py +++ b/src/autoresearch_quantum/config.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import Mapping from pathlib import Path -from typing import Any, Mapping +from typing import Any import yaml diff --git a/src/autoresearch_quantum/execution/analysis.py b/src/autoresearch_quantum/execution/analysis.py index 477e891..4e164cc 100644 --- a/src/autoresearch_quantum/execution/analysis.py +++ b/src/autoresearch_quantum/execution/analysis.py @@ -1,9 +1,10 @@ from __future__ import annotations from collections import Counter +from collections.abc import Iterable from math import sqrt from statistics import fmean, pstdev -from typing import Any, Iterable +from typing import Any def local_memory_records(memory: list[str], creg_names: list[str]) -> list[dict[str, str]]: diff --git a/src/autoresearch_quantum/execution/transfer.py b/src/autoresearch_quantum/execution/transfer.py index 5b9074a..9eeb46d 100644 --- a/src/autoresearch_quantum/execution/transfer.py +++ b/src/autoresearch_quantum/execution/transfer.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from dataclasses import replace from statistics import fmean, stdev from ..models import ExperimentSpec, RungConfig, TransferReport diff --git a/src/autoresearch_quantum/lessons/feedback.py b/src/autoresearch_quantum/lessons/feedback.py index 6fc095b..1f7095f 100644 --- a/src/autoresearch_quantum/lessons/feedback.py +++ b/src/autoresearch_quantum/lessons/feedback.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections import defaultdict from itertools import combinations -from statistics import fmean, stdev +from statistics import fmean from typing import Any from ..models import LessonFeedback, SearchRule, SearchSpaceConfig diff --git a/src/autoresearch_quantum/models.py b/src/autoresearch_quantum/models.py index 7f065b0..5f27ae7 100644 --- a/src/autoresearch_quantum/models.py +++ b/src/autoresearch_quantum/models.py @@ -1,13 +1,13 @@ from __future__ import annotations from dataclasses import asdict, dataclass, field, replace -from datetime import datetime, timezone +from datetime import UTC, datetime from hashlib import sha1 from typing import Any def utc_timestamp() -> str: - return datetime.now(timezone.utc).isoformat() + return datetime.now(UTC).isoformat() def short_hash(payload: str, length: int = 10) -> str: @@ -33,7 +33,7 @@ class ExperimentSpec: repeats: int = 3 notes: str = "" - def with_updates(self, **changes: Any) -> "ExperimentSpec": + def with_updates(self, **changes: Any) -> ExperimentSpec: if "initial_layout" in changes and isinstance(changes["initial_layout"], list): changes["initial_layout"] = tuple(changes["initial_layout"]) return replace(self, **changes) diff --git a/src/autoresearch_quantum/ratchet/runner.py b/src/autoresearch_quantum/ratchet/runner.py index cc8e52e..54b7099 100644 --- a/src/autoresearch_quantum/ratchet/runner.py +++ b/src/autoresearch_quantum/ratchet/runner.py @@ -1,8 +1,7 @@ from __future__ import annotations -import json import logging -from dataclasses import asdict, replace +from dataclasses import replace from typing import Any from ..execution.local import LocalCheapExecutor @@ -19,7 +18,7 @@ from ..models import ( generate_experiment_id, ) from ..persistence.store import ResearchStore -from ..search.challengers import GeneratedChallenger, mutation_summary +from ..search.challengers import mutation_summary from ..search.strategies import CompositeGenerator, default_composite logger = logging.getLogger(__name__) diff --git a/src/autoresearch_quantum/scoring/score.py b/src/autoresearch_quantum/scoring/score.py index 98bddad..cc9d371 100644 --- a/src/autoresearch_quantum/scoring/score.py +++ b/src/autoresearch_quantum/scoring/score.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Callable +from collections.abc import Callable from ..models import EvaluationMetrics, FactoryMetrics, QualityWeights, ScoreConfig diff --git a/src/autoresearch_quantum/teaching/__init__.py b/src/autoresearch_quantum/teaching/__init__.py index cc0ae76..610e0d4 100644 --- a/src/autoresearch_quantum/teaching/__init__.py +++ b/src/autoresearch_quantum/teaching/__init__.py @@ -1,21 +1,21 @@ """Teaching toolkit — widget-based active learning and progress tracking.""" -from autoresearch_quantum.teaching.tracker import LearningTracker from autoresearch_quantum.teaching.assess import ( - quiz, - predict_choice, - reflect, - order, - checkpoint_summary, - # Legacy aliases for backwards compatibility - multiple_choice, - predict, check_prediction, - numerical_answer, - free_response, + checkpoint_summary, code_challenge, concept_sort, + free_response, + # Legacy aliases for backwards compatibility + multiple_choice, + numerical_answer, + order, + predict, + predict_choice, + quiz, + reflect, ) +from autoresearch_quantum.teaching.tracker import LearningTracker __all__ = [ "LearningTracker", diff --git a/src/autoresearch_quantum/teaching/assess.py b/src/autoresearch_quantum/teaching/assess.py index 95e973b..1a934e8 100644 --- a/src/autoresearch_quantum/teaching/assess.py +++ b/src/autoresearch_quantum/teaching/assess.py @@ -1,7 +1,8 @@ """Assessment primitives — widget-based, click-to-answer, visually distinct from code.""" from __future__ import annotations -from typing import Any, Sequence +import warnings +from typing import Any import ipywidgets as widgets from IPython.display import HTML, display @@ -314,7 +315,6 @@ def order( tie_map[item] = gi # For each position, check the student's item is valid # Items in the same tie group can swap positions with each other - used = list(correct_order) for i, s_item in enumerate(student): c_item = correct_order[i] if s_item == c_item: @@ -413,29 +413,49 @@ def multiple_choice(tracker, qid, question, options, correct, answer="?", quiz(tracker, qid, question, opt_list, correct_idx, bloom, explanation) def predict(tracker, qid, question, your_prediction="?", bloom="understand"): - """Legacy wrapper — now a no-op since predict_choice replaces the pair.""" - pass + """Legacy wrapper — use predict_choice() instead.""" + warnings.warn( + "predict() is deprecated and does nothing. Use predict_choice() instead.", + DeprecationWarning, stacklevel=2, + ) def check_prediction(tracker, qid, actual_value=None, was_correct=False, explanation=""): - """Legacy wrapper — no-op.""" - pass + """Legacy wrapper — use predict_choice() instead.""" + warnings.warn( + "check_prediction() is deprecated and does nothing. Use predict_choice() instead.", + DeprecationWarning, stacklevel=2, + ) def numerical_answer(tracker, qid, question, answer=0.0, correct=0.0, tolerance=0.01, bloom="apply", explanation=""): - """Legacy wrapper — converts to MCQ with the correct value and distractors.""" - pass + """Legacy wrapper — use quiz() instead.""" + warnings.warn( + "numerical_answer() is deprecated and does nothing. Use quiz() instead.", + DeprecationWarning, stacklevel=2, + ) def free_response(tracker, qid, question, answer="?", bloom="evaluate", model_answer=""): """Legacy wrapper — redirects to reflect().""" + warnings.warn( + "free_response() is deprecated. Use reflect() directly.", + DeprecationWarning, stacklevel=2, + ) reflect(tracker, qid, question, model_answer, bloom) def code_challenge(tracker, qid, description, test_passed=False, bloom="apply", hint="", explanation=""): - """Legacy wrapper — no-op (code challenges stay as code cells).""" - pass + """Legacy wrapper — no replacement; use code cells with assertions.""" + warnings.warn( + "code_challenge() is deprecated and does nothing. Use code cells with assertions.", + DeprecationWarning, stacklevel=2, + ) def concept_sort(tracker, qid, instruction, student_order=None, correct_order=None, bloom="analyze", explanation=""): - """Legacy wrapper — redirects to order().""" + """Legacy wrapper — use order() instead.""" + warnings.warn( + "concept_sort() is deprecated. Use order() directly.", + DeprecationWarning, stacklevel=2, + ) if correct_order: order(tracker, qid, instruction, list(correct_order), list(correct_order), bloom, explanation) diff --git a/src/autoresearch_quantum/teaching/tracker.py b/src/autoresearch_quantum/teaching/tracker.py index 98aef8a..27c875c 100644 --- a/src/autoresearch_quantum/teaching/tracker.py +++ b/src/autoresearch_quantum/teaching/tracker.py @@ -10,7 +10,6 @@ from typing import Any from IPython.display import HTML, display - # Bloom's taxonomy levels, ordered by cognitive demand BLOOM_LEVELS = ["remember", "understand", "apply", "analyze", "evaluate", "create"] @@ -89,35 +88,28 @@ class LearningTracker: self.attempts.append(attempt) # ── queries ───────────────────────────────────────────────────────── - def score_by_section(self) -> dict[str, dict[str, int]]: - """Returns {section: {correct: n, incorrect: n, total: n, pct: float}}.""" - sections: dict[str, dict[str, int]] = defaultdict(lambda: {"correct": 0, "incorrect": 0, "total": 0}) - seen: set[str] = set() - for a in self.attempts: - if a.correct is None: - continue - # only count the *latest* attempt for each question - key = a.question_id - if key in seen: - # remove previous count - continue - seen.add(key) - # recount with latest attempts only + def score_by_section(self) -> dict[str, dict[str, Any]]: + """Returns {section: {correct: n, incorrect: n, total: n, pct: float}}. + + Only the latest attempt per question is counted. + """ latest: dict[str, Attempt] = {} for a in self.attempts: if a.correct is not None: latest[a.question_id] = a - sections_out: dict[str, dict[str, Any]] = defaultdict(lambda: {"correct": 0, "incorrect": 0, "total": 0}) + sections: dict[str, dict[str, Any]] = defaultdict( + lambda: {"correct": 0, "incorrect": 0, "total": 0}, + ) for a in latest.values(): - s = sections_out[a.section] + s = sections[a.section] s["total"] += 1 if a.correct: s["correct"] += 1 else: s["incorrect"] += 1 - for s in sections_out.values(): + for s in sections.values(): s["pct"] = round(100 * s["correct"] / s["total"], 1) if s["total"] else 0.0 - return dict(sections_out) + return dict(sections) def score_by_bloom(self) -> dict[str, dict[str, Any]]: """Returns {bloom_level: {correct: n, total: n, pct: float}}.""" diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 71f0e66..6332fb2 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -14,7 +14,6 @@ from autoresearch_quantum.execution.analysis import ( syndrome_outcomes, ) - # ── local_memory_records ───────────────────────────────────────────────────── def test_local_memory_records_splits_correctly() -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 452cc41..d47d97d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,7 +5,6 @@ import pytest from autoresearch_quantum.cli import _parse_override, build_parser - # ── _parse_override ────────────────────────────────────────────────────────── def test_parse_override_bool_true() -> None: diff --git a/tests/test_codes.py b/tests/test_codes.py index 923e759..3578445 100644 --- a/tests/test_codes.py +++ b/tests/test_codes.py @@ -1,9 +1,8 @@ """Tests for codes.four_two_two — all seed and encoder styles.""" from __future__ import annotations -from math import pi - import pytest +from qiskit import QuantumCircuit from qiskit.quantum_info import Statevector from autoresearch_quantum.codes.four_two_two import ( @@ -11,10 +10,7 @@ from autoresearch_quantum.codes.four_two_two import ( apply_magic_seed, build_encoder, build_preparation_circuit, - encoded_magic_statevector, ) -from qiskit import QuantumCircuit - # ── apply_magic_seed ───────────────────────────────────────────────────────── diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 909daef..a89e46f 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -2,7 +2,6 @@ from __future__ import annotations from autoresearch_quantum.lessons.feedback import ( - build_lesson_feedback, extract_search_rules, narrow_search_space, ) diff --git a/tests/test_harness.py b/tests/test_harness.py index 7359d2e..fe8e8bd 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -5,9 +5,9 @@ from pathlib import Path from qiskit.quantum_info import Statevector from autoresearch_quantum.codes.four_two_two import STABILIZERS, encoded_magic_statevector -from autoresearch_quantum.experiments.encoded_magic_state import build_circuit_bundle from autoresearch_quantum.execution.local import LocalCheapExecutor from autoresearch_quantum.execution.transfer import TransferEvaluator +from autoresearch_quantum.experiments.encoded_magic_state import build_circuit_bundle from autoresearch_quantum.lessons.feedback import ( build_lesson_feedback, extract_search_rules, @@ -16,7 +16,6 @@ from autoresearch_quantum.lessons.feedback import ( from autoresearch_quantum.models import ( CostWeights, ExperimentSpec, - FactoryMetrics, HardwareConfig, LessonFeedback, QualityWeights, @@ -30,14 +29,11 @@ from autoresearch_quantum.models import ( ) from autoresearch_quantum.persistence.store import ResearchStore from autoresearch_quantum.ratchet.runner import AutoresearchHarness -from autoresearch_quantum.scoring.score import factory_throughput_score, score_metrics +from autoresearch_quantum.scoring.score import factory_throughput_score from autoresearch_quantum.search.challengers import generate_neighbor_challengers from autoresearch_quantum.search.strategies import ( - CompositeGenerator, LessonGuided, - NeighborWalk, RandomCombo, - StrategyWeight, default_composite, ) diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py new file mode 100644 index 0000000..88126fc --- /dev/null +++ b/tests/test_notebooks.py @@ -0,0 +1,126 @@ +"""Notebook execution tests — validates that all notebooks run without errors. + +Uses nbclient to execute each notebook in a fresh kernel. This catches: +- Import errors +- Broken code cells +- Widget rendering failures +- API mismatches between notebooks and library code +""" +from __future__ import annotations + +import re +import warnings +from pathlib import Path + +import nbformat +import pytest +from nbclient import NotebookClient +from nbclient.exceptions import CellExecutionError + +NOTEBOOK_DIR = Path("notebooks") + +ALL_NOTEBOOKS = sorted(NOTEBOOK_DIR.rglob("*.ipynb")) + +# Execution timeout per cell (seconds) — noisy simulation cells can be slow +CELL_TIMEOUT = 180 + + +def _notebook_id(path: Path) -> str: + """Create a readable test ID from notebook path.""" + return str(path.relative_to(NOTEBOOK_DIR)).replace("/", "__").removesuffix(".ipynb") + + +@pytest.fixture(params=ALL_NOTEBOOKS, ids=[_notebook_id(p) for p in ALL_NOTEBOOKS]) +def notebook_path(request): + return request.param + + +def test_notebook_valid_json(notebook_path): + """Notebook file is valid JSON and nbformat.""" + nb = nbformat.read(str(notebook_path), as_version=4) + assert nb.nbformat == 4 + assert len(nb.cells) > 0 + + +def test_notebook_has_code_cells(notebook_path): + """Every notebook has at least one code cell.""" + nb = nbformat.read(str(notebook_path), as_version=4) + code_cells = [c for c in nb.cells if c.cell_type == "code"] + assert len(code_cells) > 0, f"{notebook_path} has no code cells" + + +def test_notebook_executes(notebook_path): + """Execute the full notebook in a fresh kernel — no cell may raise.""" + nb = nbformat.read(str(notebook_path), as_version=4) + # Normalize to add missing cell IDs (avoids nbformat warnings) + _, nb = nbformat.validator.normalize(nb) + + # Set cwd to the notebook's directory so relative paths (e.g. ../../configs) + # resolve correctly, matching how a student would run the notebook. + nb_dir = str(notebook_path.resolve().parent) + client = NotebookClient( + nb, + timeout=CELL_TIMEOUT, + kernel_name="python3", + resources={"metadata": {"path": nb_dir}}, + ) + try: + client.execute() + except CellExecutionError as exc: + # Extract the failing cell for a clear error message + pytest.fail( + f"Notebook {notebook_path} failed during execution:\n" + f"Cell index: {exc.cell_index if hasattr(exc, 'cell_index') else '?'}\n" + f"{exc}" + ) + + +def test_notebook_quiz_cells_have_section(notebook_path): + """Every quiz/predict/reflect/order call should specify a section= parameter. + + Without a section, tracker scores are all lumped into 'intro', making + checkpoint_summary useless. + """ + nb = nbformat.read(str(notebook_path), as_version=4) + missing_section = [] + for i, cell in enumerate(nb.cells): + if cell.cell_type != "code": + continue + src = "".join(cell.source) + for fn in ["quiz(", "predict_choice(", "reflect(", "order("]: + if fn in src and "section=" not in src and "tracker" in src and "LearningTracker" not in src: + missing_section.append((i, fn.rstrip("("))) + if missing_section: + details = ", ".join(f"cell {i} ({fn})" for i, fn in missing_section) + warnings.warn( + f"{notebook_path}: {len(missing_section)} assessment calls missing section= parameter: {details}", + UserWarning, + stacklevel=1, + ) + + +def test_learning_objectives_document_exists(): + """learning_objectives.md exists and covers all four plans.""" + obj_path = NOTEBOOK_DIR / "learning_objectives.md" + assert obj_path.exists(), "learning_objectives.md not found" + text = obj_path.read_text() + assert "Plan A" in text + assert "Plan B" in text + assert "Plan C" in text + assert "Plan D" in text + assert "four plans" in text.lower() + + +def test_every_notebook_has_assessments(): + """Every notebook has at least one assessment cell (quiz/predict/reflect/order).""" + assessment_pattern = re.compile(r"(quiz|predict_choice|reflect|order)\s*\(") + for nb_path in ALL_NOTEBOOKS: + nb = nbformat.read(str(nb_path), as_version=4) + has_assessment = False + for cell in nb.cells: + if cell.cell_type == "code": + src = "".join(cell.source) + if assessment_pattern.search(src) and "LearningTracker" not in src: + has_assessment = True + break + assert has_assessment, f"{nb_path} has no assessment cells" diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 286e9f5..fb4c5ea 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -5,7 +5,6 @@ import pytest from autoresearch_quantum.models import EvaluationMetrics, QualityWeights, ScoreConfig from autoresearch_quantum.scoring.score import ( - SCORE_REGISTRY, score_metrics, weighted_acceptance_cost, ) diff --git a/tests/test_teaching.py b/tests/test_teaching.py new file mode 100644 index 0000000..e764e15 --- /dev/null +++ b/tests/test_teaching.py @@ -0,0 +1,441 @@ +"""Tests for the teaching layer: assess.py and tracker.py.""" +from __future__ import annotations + +import json +import warnings +from unittest.mock import MagicMock, patch + +import pytest + +from autoresearch_quantum.teaching.tracker import LearningTracker + +# ============================================================================ +# LearningTracker — core logic +# ============================================================================ + +class TestLearningTracker: + def test_init(self): + t = LearningTracker("test_nb") + assert t.notebook_id == "test_nb" + assert t.attempts == [] + assert t.current_section == "intro" + + def test_set_section(self): + t = LearningTracker("test_nb") + t.set_section("stabilisers") + assert t.current_section == "stabilisers" + + def test_record_appends_attempt(self): + t = LearningTracker("test_nb") + t.set_section("sec1") + t.record("q1", "remember", True, "A", "A") + assert len(t.attempts) == 1 + a = t.attempts[0] + assert a.question_id == "q1" + assert a.bloom_level == "remember" + assert a.correct is True + assert a.section == "sec1" + assert a.attempt_number == 1 + + def test_record_increments_attempt_number(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", False, "B", "A") + t.record("q1", "remember", True, "A", "A") + assert t.attempts[0].attempt_number == 1 + assert t.attempts[1].attempt_number == 2 + + def test_record_none_correct_for_reflections(self): + t = LearningTracker("test_nb") + t.record("r1", "evaluate", None, "my thoughts", "model answer") + assert t.attempts[0].correct is None + + +class TestScoreBySection: + def test_empty_tracker(self): + t = LearningTracker("test_nb") + assert t.score_by_section() == {} + + def test_single_correct(self): + t = LearningTracker("test_nb") + t.set_section("sec1") + t.record("q1", "remember", True, "A", "A") + result = t.score_by_section() + assert result["sec1"]["correct"] == 1 + assert result["sec1"]["total"] == 1 + assert result["sec1"]["pct"] == 100.0 + + def test_mixed_results(self): + t = LearningTracker("test_nb") + t.set_section("sec1") + t.record("q1", "remember", True, "A", "A") + t.record("q2", "understand", False, "B", "A") + result = t.score_by_section() + assert result["sec1"]["correct"] == 1 + assert result["sec1"]["incorrect"] == 1 + assert result["sec1"]["total"] == 2 + assert result["sec1"]["pct"] == 50.0 + + def test_latest_attempt_wins(self): + """Only the latest attempt per question should count.""" + t = LearningTracker("test_nb") + t.set_section("sec1") + t.record("q1", "remember", False, "B", "A") + t.record("q1", "remember", True, "A", "A") + result = t.score_by_section() + assert result["sec1"]["correct"] == 1 + assert result["sec1"]["incorrect"] == 0 + assert result["sec1"]["total"] == 1 + + def test_none_correct_excluded(self): + """Reflections (correct=None) should not appear in section scores.""" + t = LearningTracker("test_nb") + t.set_section("sec1") + t.record("r1", "evaluate", None, "my thoughts", "model") + assert t.score_by_section() == {} + + def test_multiple_sections(self): + t = LearningTracker("test_nb") + t.set_section("sec1") + t.record("q1", "remember", True, "A", "A") + t.set_section("sec2") + t.record("q2", "apply", False, "B", "A") + result = t.score_by_section() + assert "sec1" in result + assert "sec2" in result + assert result["sec1"]["pct"] == 100.0 + assert result["sec2"]["pct"] == 0.0 + + +class TestScoreByBloom: + def test_groups_by_bloom(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", True, "A", "A") + t.record("q2", "apply", False, "B", "A") + result = t.score_by_bloom() + assert result["remember"]["correct"] == 1 + assert result["apply"]["correct"] == 0 + + +class TestStruggledQuestions: + def test_no_struggles(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", True, "A", "A") + assert t.struggled_questions() == [] + + def test_wrong_answer_is_struggled(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", False, "B", "A") + assert "q1" in t.struggled_questions() + + def test_multiple_attempts_is_struggled(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", False, "B", "A") + t.record("q1", "remember", True, "A", "A") + assert "q1" in t.struggled_questions() + + +class TestMasteryScore: + def test_empty(self): + t = LearningTracker("test_nb") + assert t.mastery_score() == 0.0 + + def test_perfect(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", True, "A", "A") + t.record("q2", "understand", True, "B", "B") + assert t.mastery_score() == 100.0 + + def test_partial(self): + t = LearningTracker("test_nb") + t.record("q1", "remember", True, "A", "A") + t.record("q2", "understand", False, "B", "A") + assert t.mastery_score() == 50.0 + + +class TestSave: + def test_save_to_explicit_path(self, tmp_path): + t = LearningTracker("test_nb") + t.record("q1", "remember", True, "A", "A") + out = t.save(tmp_path / "progress.json") + assert out.exists() + data = json.loads(out.read_text()) + assert data["notebook_id"] == "test_nb" + assert data["mastery_score"] == 100.0 + assert len(data["attempts"]) == 1 + + def test_save_to_save_dir(self, tmp_path): + t = LearningTracker("test_nb", save_dir=tmp_path) + t.record("q1", "remember", True, "A", "A") + out = t.save() + assert out == tmp_path / "test_nb_progress.json" + assert out.exists() + + def test_save_default_cwd(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + t = LearningTracker("test_nb") + t.record("q1", "remember", True, "A", "A") + out = t.save() + assert out.name == "test_nb_progress.json" + assert out.exists() + + +# ============================================================================ +# assess.py — widget functions +# ============================================================================ + +# We mock ipywidgets and IPython.display since we're not running in a notebook. + +@pytest.fixture +def mock_display(): + """Mock IPython display to capture display calls.""" + with patch("autoresearch_quantum.teaching.assess.display") as mock_d: + yield mock_d + + +@pytest.fixture +def mock_widgets(): + """Mock ipywidgets so we can test widget construction logic.""" + with patch("autoresearch_quantum.teaching.assess.widgets") as mock_w: + # Make VBox return a MagicMock that we can track + mock_w.VBox.return_value = MagicMock(name="VBox") + mock_w.HTML.return_value = MagicMock(name="HTML") + mock_w.RadioButtons.return_value = MagicMock(name="RadioButtons", value=None) + mock_w.Button.return_value = MagicMock(name="Button") + mock_w.Textarea.return_value = MagicMock(name="Textarea", value="") + mock_w.Dropdown.return_value = MagicMock(name="Dropdown", value="(select)") + mock_w.Layout.return_value = MagicMock(name="Layout") + yield mock_w + + +class TestQuiz: + def test_returns_none(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import quiz + t = LearningTracker("test") + result = quiz(t, "q1", "What?", ["A", "B"], 0, section="s1") + assert result is None + + def test_displays_exactly_once(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import quiz + t = LearningTracker("test") + quiz(t, "q1", "What?", ["A", "B"], 0) + assert mock_display.call_count == 1 + + def test_sets_section(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import quiz + t = LearningTracker("test") + quiz(t, "q1", "What?", ["A", "B"], 0, section="mysec") + assert t.current_section == "mysec" + + def test_submit_correct_records(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import quiz + t = LearningTracker("test") + mock_btn = mock_widgets.Button.return_value + mock_radio = mock_widgets.RadioButtons.return_value + + quiz(t, "q1", "What?", ["A", "B"], 0, bloom="remember") + + # Simulate user selecting correct answer and clicking submit + mock_radio.value = "A" + # Get the on_click callback + on_click_call = mock_btn.on_click.call_args + callback = on_click_call[0][0] + callback(None) + + assert len(t.attempts) == 1 + assert t.attempts[0].correct is True + assert t.attempts[0].student_answer == "A" + + def test_submit_incorrect_records(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import quiz + t = LearningTracker("test") + mock_btn = mock_widgets.Button.return_value + mock_radio = mock_widgets.RadioButtons.return_value + + quiz(t, "q1", "What?", ["A", "B"], 0, bloom="remember") + + mock_radio.value = "B" + callback = mock_btn.on_click.call_args[0][0] + callback(None) + + assert t.attempts[0].correct is False + + +class TestPredictChoice: + def test_returns_none(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import predict_choice + t = LearningTracker("test") + result = predict_choice(t, "q1", "Predict?", ["A", "B"], 0) + assert result is None + + def test_displays_exactly_once(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import predict_choice + t = LearningTracker("test") + predict_choice(t, "q1", "Predict?", ["A", "B"], 0) + assert mock_display.call_count == 1 + + +class TestReflect: + def test_returns_none(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import reflect + t = LearningTracker("test") + result = reflect(t, "r1", "What do you think?") + assert result is None + + def test_displays_exactly_once(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import reflect + t = LearningTracker("test") + reflect(t, "r1", "What do you think?") + assert mock_display.call_count == 1 + + +class TestOrder: + def test_returns_none(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import order + t = LearningTracker("test") + result = order(t, "o1", "Sort these:", ["A", "B"], ["A", "B"]) + assert result is None + + def test_displays_exactly_once(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import order + t = LearningTracker("test") + order(t, "o1", "Sort:", ["A", "B"], ["A", "B"]) + assert mock_display.call_count == 1 + + def test_order_check_exact(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import order + t = LearningTracker("test") + + # We need to test the _check_order logic. Since it's nested, + # we test via the submit callback. + mock_btn = mock_widgets.Button.return_value + + # Create two dropdowns that will return values + dd1 = MagicMock(value="A", disabled=False) + dd2 = MagicMock(value="B", disabled=False) + mock_widgets.Dropdown.side_effect = [dd1, dd2] + + order(t, "o1", "Sort:", ["A", "B"], ["A", "B"]) + + callback = mock_btn.on_click.call_args[0][0] + callback(None) + + assert len(t.attempts) == 1 + assert t.attempts[0].correct is True + + def test_order_check_with_ties(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import order + t = LearningTracker("test") + mock_btn = mock_widgets.Button.return_value + + # Student puts Z before X (tied, so should be correct) + dd1 = MagicMock(value="Z", disabled=False) + dd2 = MagicMock(value="X", disabled=False) + dd3 = MagicMock(value="Y", disabled=False) + mock_widgets.Dropdown.side_effect = [dd1, dd2, dd3] + + order(t, "o1", "Sort:", ["X", "Z", "Y"], ["X", "Z", "Y"], + ties=[["X", "Z"]]) + + callback = mock_btn.on_click.call_args[0][0] + callback(None) + + assert t.attempts[0].correct is True + + def test_order_check_wrong(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import order + t = LearningTracker("test") + mock_btn = mock_widgets.Button.return_value + + dd1 = MagicMock(value="Y", disabled=False) + dd2 = MagicMock(value="X", disabled=False) + dd3 = MagicMock(value="Z", disabled=False) + mock_widgets.Dropdown.side_effect = [dd1, dd2, dd3] + + order(t, "o1", "Sort:", ["X", "Z", "Y"], ["X", "Z", "Y"]) + + callback = mock_btn.on_click.call_args[0][0] + callback(None) + + assert t.attempts[0].correct is False + + +class TestCheckpointSummary: + def test_displays_html(self, mock_display): + with patch("autoresearch_quantum.teaching.assess.HTML"): + from autoresearch_quantum.teaching.assess import checkpoint_summary + t = LearningTracker("test") + t.set_section("sec1") + t.record("q1", "remember", True, "A", "A") + checkpoint_summary(t, "sec1") + assert mock_display.call_count == 1 + + +# ============================================================================ +# Legacy wrappers — deprecation warnings +# ============================================================================ + +class TestLegacyWrappers: + def test_predict_warns(self): + from autoresearch_quantum.teaching.assess import predict + t = LearningTracker("test") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + predict(t, "q1", "What?") + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert "predict_choice" in str(w[0].message) + + def test_check_prediction_warns(self): + from autoresearch_quantum.teaching.assess import check_prediction + t = LearningTracker("test") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + check_prediction(t, "q1") + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + + def test_numerical_answer_warns(self): + from autoresearch_quantum.teaching.assess import numerical_answer + t = LearningTracker("test") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + numerical_answer(t, "q1", "How many?") + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + + def test_code_challenge_warns(self): + from autoresearch_quantum.teaching.assess import code_challenge + t = LearningTracker("test") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + code_challenge(t, "q1", "Do X") + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + + def test_free_response_warns_and_redirects(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import free_response + t = LearningTracker("test") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + free_response(t, "q1", "Explain?", model_answer="Because X") + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + # Should still display a reflect widget + assert mock_display.call_count == 1 + + def test_concept_sort_warns_and_redirects(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import concept_sort + t = LearningTracker("test") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + concept_sort(t, "q1", "Sort:", correct_order=["A", "B"]) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert mock_display.call_count == 1 + + def test_multiple_choice_redirects(self, mock_display, mock_widgets): + from autoresearch_quantum.teaching.assess import multiple_choice + t = LearningTracker("test") + # multiple_choice is not deprecated (it works), just redirects + multiple_choice(t, "q1", "What?", {"a": "Alpha", "b": "Beta"}, "a") + assert mock_display.call_count == 1