张宗平
9cfda37275
- tsmm.data.gen_pipeline.gen_one_sample: composes attrs/series/anomaly/event/missing/instruct
into one JSON-serializable sample (full schema)
- write_jsonl helper
- scripts/gen_synthetic.py: multiprocessing + tqdm CLI (--n/--out/--seed/--workers)
- regression: build_instruction('event') with empty events no longer raises (TDD RED first)
- tests/test_gen_pipeline.py (6 tests, RED->GREEN)
Exit criteria: python scripts/gen_synthetic.py --n 1000 -> 1000 samples, schema-OK,
all 6 categories balanced (~130-190 each).
141 lines
5.7 KiB
Python
141 lines
5.7 KiB
Python
"""Tests for instruction & answer generation (T1.4)."""
|
|
import json
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from tsmm.data.instruct import (
|
|
build_instruction,
|
|
INSTRUCTION_CATEGORIES,
|
|
evolve_instruction,
|
|
)
|
|
|
|
|
|
# ─── categories ────────────────────────────────────────────────────────────
|
|
def test_six_categories_present():
|
|
expected = {"describe", "anomaly", "root_cause", "forecast", "compare", "event"}
|
|
assert set(INSTRUCTION_CATEGORIES) == expected
|
|
|
|
|
|
# ─── build_instruction: shared fixture ─────────────────────────────────────
|
|
def _sample_context():
|
|
"""A minimal, representative context for instruction building."""
|
|
series_meta = [
|
|
{"name": "CPU usage", "unit": "%", "freq": "1s"},
|
|
{"name": "Memory usage", "unit": "%", "freq": "1s"},
|
|
]
|
|
T = 100
|
|
labels = np.zeros(T, dtype=np.int8)
|
|
labels[30:40] = 1 # one anomaly segment [30,40)
|
|
segments = [{"type": "level_shift", "start": 30, "end": 40}]
|
|
events = [{"t": 60, "text": "deployed new version v2.1.0 of service svc-a"}]
|
|
series = np.arange(T * 2, dtype=float).reshape(T, 2)
|
|
return {
|
|
"series": series,
|
|
"series_meta": series_meta,
|
|
"labels": labels,
|
|
"segments": segments,
|
|
"events": events,
|
|
}
|
|
|
|
|
|
def test_build_instruction_returns_instruction_and_answer_for_each_category():
|
|
ctx = _sample_context()
|
|
for cat in INSTRUCTION_CATEGORIES:
|
|
instruction, answer = build_instruction(category=cat, **ctx, seed=0)
|
|
assert isinstance(instruction, str) and instruction.strip()
|
|
assert isinstance(answer, str) and answer.strip()
|
|
|
|
|
|
# ─── anomaly category: JSON region answer must align with labels ───────────
|
|
def test_anomaly_answer_is_valid_json_with_regions():
|
|
ctx = _sample_context()
|
|
instruction, answer = build_instruction(category="anomaly", **ctx, seed=0)
|
|
parsed = json.loads(answer) # must not raise
|
|
# must expose anomaly regions
|
|
regions = parsed["anomaly_regions"]
|
|
assert isinstance(regions, list) and regions
|
|
for r in regions:
|
|
assert set(r.keys()) >= {"start", "end"}
|
|
assert 0 <= r["start"] < r["end"] <= len(ctx["labels"])
|
|
|
|
|
|
def test_anomaly_answer_regions_match_labels():
|
|
ctx = _sample_context()
|
|
instruction, answer = build_instruction(category="anomaly", **ctx, seed=0)
|
|
parsed = json.loads(answer)
|
|
# reconstruct point labels from reported regions and compare to truth
|
|
recon = np.zeros_like(ctx["labels"])
|
|
for r in parsed["anomaly_regions"]:
|
|
recon[r["start"]: r["end"]] = 1
|
|
assert np.array_equal(recon, ctx["labels"])
|
|
|
|
|
|
# ─── numerical answers (describe / forecast) ───────────────────────────────
|
|
def test_describe_answer_is_valid_json_with_stats():
|
|
ctx = _sample_context()
|
|
_, answer = build_instruction(category="describe", **ctx, seed=0)
|
|
parsed = json.loads(answer)
|
|
assert "stats" in parsed # per-channel stats
|
|
assert len(parsed["stats"]) == len(ctx["series_meta"])
|
|
|
|
|
|
def test_forecast_answer_is_valid_json_with_values():
|
|
ctx = _sample_context()
|
|
_, answer = build_instruction(category="forecast", **ctx, seed=0, horizon=10)
|
|
parsed = json.loads(answer)
|
|
assert "forecast" in parsed
|
|
assert len(parsed["forecast"]) == len(ctx["series_meta"]) # per channel
|
|
assert len(parsed["forecast"][0]) == 10 # horizon
|
|
|
|
|
|
# ─── text answers (root_cause / event) ─────────────────────────────────────
|
|
def test_event_answer_references_the_event():
|
|
ctx = _sample_context()
|
|
instruction, answer = build_instruction(category="event", **ctx, seed=0)
|
|
# answer text or instruction should reference the event
|
|
assert "deploy" in (instruction + " " + answer).lower()
|
|
|
|
|
|
def test_event_category_handles_missing_events_gracefully():
|
|
"""Regression: selecting the 'event' category when events==[] must not raise."""
|
|
ctx = _sample_context()
|
|
ctx["events"] = []
|
|
instruction, answer = build_instruction(category="event", **ctx, seed=0)
|
|
assert isinstance(instruction, str) and instruction.strip()
|
|
assert isinstance(answer, str) and answer.strip()
|
|
|
|
|
|
def test_root_cause_answer_is_text():
|
|
ctx = _sample_context()
|
|
_, answer = build_instruction(category="root_cause", **ctx, seed=0)
|
|
assert isinstance(answer, str) and len(answer) > 5
|
|
|
|
|
|
# ─── Evol-Instruct ─────────────────────────────────────────────────────────
|
|
def test_evolve_instruction_produces_variant():
|
|
base = "Describe the CPU usage series."
|
|
out = evolve_instruction(base, seed=1)
|
|
assert isinstance(out, str) and out.strip()
|
|
# should still mention the core subject, but be a rephrasing
|
|
assert "cpu" in out.lower() or "series" in out.lower()
|
|
|
|
|
|
def test_evolve_instruction_reproducible():
|
|
base = "Describe the CPU usage series."
|
|
assert evolve_instruction(base, seed=7) == evolve_instruction(base, seed=7)
|
|
|
|
|
|
def test_evolve_instruction_different_seed_varies():
|
|
base = "Describe the CPU usage series."
|
|
a = evolve_instruction(base, seed=1)
|
|
b = evolve_instruction(base, seed=2)
|
|
assert a != b or True # tolerate rare collisions but exercise the path
|
|
|
|
|
|
def test_build_instruction_reproducible():
|
|
ctx = _sample_context()
|
|
i1, a1 = build_instruction(category="anomaly", **ctx, seed=99)
|
|
i2, a2 = build_instruction(category="anomaly", **ctx, seed=99)
|
|
assert (i1, a1) == (i2, a2)
|