#!/usr/bin/env bash
# M4 + M5 exit-validation eval (T4.2 / T4.3 / T5.6).
#
# Runs (1) the instruction-following check (T4.2) and (2) the full eval report
# (T5.5/5.6) including the trained model + trivial baselines, on the stage-2
# checkpoint. Requires a free GPU (so run AFTER training finishes).
#
# Usage:
#   scripts/run_m4_m5_eval.sh [stage2_ckpt]
set -euo pipefail
cd "$(dirname "$0")/.."
export PYTHONPATH=src

CKPT="${1:-checkpoints/stage2/stage2_final.pt}"
if [ ! -f "$CKPT" ]; then
  echo "[run_m4_m5_eval] checkpoint $CKPT not found"; exit 1
fi

echo "=== T4.2 instruction-following check (6 cats × 10) ==="
.venv/bin/python -m tsmm.eval.instruct_check \
  --data data/eval_synth.jsonl \
  --per_cat 10 \
  --stage2_ckpt "$CKPT" \
  --judge_backend stub \
  --out reports/instruct_check.json

echo
echo "=== T5.5/T5.6 full eval report (model + trivial baselines) ==="
.venv/bin/python -m tsmm.eval.report \
  --synth data/eval_synth.jsonl \
  --model \
  --stage2_ckpt "$CKPT" \
  --judge_backend stub \
  --report_dir reports

echo
echo "[run_m4_m5_eval] done. See reports/."