diff --git a/scripts/train_stage1.sh b/scripts/train_stage1.sh index a8e6567..819db4a 100755 --- a/scripts/train_stage1.sh +++ b/scripts/train_stage1.sh @@ -8,6 +8,7 @@ MAX_STEPS="${2:-10000}" CKPT_DIR="${3:-checkpoints/stage1}" cd "$(dirname "$0")/.." +export PYTHONPATH=src PYTHONUNBUFFERED=1 .venv/bin/python -m tsmm.train.stage1 \ --data "$DATA" \ --ckpt_dir "$CKPT_DIR" \ diff --git a/scripts/train_stage2.sh b/scripts/train_stage2.sh index 55c57c8..d38f3c3 100755 --- a/scripts/train_stage2.sh +++ b/scripts/train_stage2.sh @@ -8,6 +8,7 @@ STAGE1_CKPT="${3:-checkpoints/stage1/stage1_final.pt}" CKPT_DIR="${4:-checkpoints/stage2}" cd "$(dirname "$0")/.." +export PYTHONPATH=src PYTHONUNBUFFERED=1 .venv/bin/python -m tsmm.train.stage2 \ --data "$DATA" \ --stage1_ckpt "$STAGE1_CKPT" \ diff --git a/tests/test_instruct_check.py b/tests/test_instruct_check.py new file mode 100644 index 0000000..41d76b9 --- /dev/null +++ b/tests/test_instruct_check.py @@ -0,0 +1,91 @@ +"""Unit tests for eval.instruct_check (T4.2) — CPU helpers only. + +The model-driven ``evaluate`` path is exercised live in M4 exit validation. +""" +import json +import os +import tempfile + +import pytest + +from tsmm.eval.instruct_check import ( + CATEGORIES, + _forecast_list, + load_by_category, +) + + +def _write_eval(tmpdir, rows): + path = os.path.join(tmpdir, "eval.jsonl") + with open(path, "w") as f: + for r in rows: + f.write(json.dumps(r) + "\n") + return path + + +def test_load_by_category_buckets_and_caps(tmp_path): + rows = [] + for cat in CATEGORIES: + for i in range(5): + rows.append({"category": cat, "series": [[1.0]], "answer": "x", "labels": [0]}) + # plus an unknown-category row that must be ignored + rows.append({"category": "mystery", "series": [[1.0]], "answer": "x"}) + path = _write_eval(str(tmp_path), rows) + out = load_by_category(path, per_cat=3, seed=0) + assert set(out.keys()) == set(CATEGORIES) + for cat in CATEGORIES: + assert len(out[cat]) == 3 + assert "mystery" not in out + + +def test_load_by_category_deterministic_with_seed(tmp_path): + rows = [{"category": "anomaly", "series": [[float(i)]], "answer": str(i), "labels": [0]} + for i in range(20)] + path = _write_eval(str(tmp_path), rows) + a = load_by_category(path, per_cat=5, seed=42) + b = load_by_category(path, per_cat=5, seed=42) + assert [s["answer"] for s in a["anomaly"]] == [s["answer"] for s in b["anomaly"]] + # different seed → (very likely) different selection + c = load_by_category(path, per_cat=5, seed=7) + assert [s["answer"] for s in a["anomaly"]] != [s["answer"] for s in c["anomaly"]] + + +def test_load_by_category_handles_short_pool(tmp_path): + rows = [{"category": "event", "series": [[1.0]], "answer": "x"} for _ in range(2)] + path = _write_eval(str(tmp_path), rows) + out = load_by_category(path, per_cat=10, seed=0) + assert len(out["event"]) == 2 + # missing categories are empty lists, not errors + assert out["anomaly"] == [] + + +def test_load_by_category_skips_blank_lines(tmp_path): + path = os.path.join(str(tmp_path), "eval.jsonl") + with open(path, "w") as f: + f.write(json.dumps({"category": "anomaly", "series": [[1.0]], "answer": "x"}) + "\n") + f.write("\n") + f.write(" \n") + f.write(json.dumps({"category": "anomaly", "series": [[2.0]], "answer": "y"}) + "\n") + out = load_by_category(path, per_cat=5, seed=0) + assert len(out["anomaly"]) == 2 + + +def test_forecast_list_from_nested_json(): + ref = json.dumps({"forecast": [[1.0, 2.0, 3.0]]}) + assert _forecast_list(ref) == [1.0, 2.0, 3.0] + + +def test_forecast_list_from_flat_json(): + ref = json.dumps({"forecast": [4.0, 5.0]}) + assert _forecast_list(ref) == [4.0, 5.0] + + +def test_forecast_list_accepts_dict_input(): + assert _forecast_list({"forecast": [[1.0, 2.0]]}) == [1.0, 2.0] + + +def test_forecast_list_returns_empty_on_garbage(): + assert _forecast_list("not json") == [] + assert _forecast_list(json.dumps({"no_forecast": 1})) == [] + assert _forecast_list("") == [] + assert _forecast_list(None) == []