feat(m1): offline generation pipeline + CLI (T1.5)
- tsmm.data.gen_pipeline.gen_one_sample: composes attrs/series/anomaly/event/missing/instruct
into one JSON-serializable sample (full schema)
- write_jsonl helper
- scripts/gen_synthetic.py: multiprocessing + tqdm CLI (--n/--out/--seed/--workers)
- regression: build_instruction('event') with empty events no longer raises (TDD RED first)
- tests/test_gen_pipeline.py (6 tests, RED->GREEN)
Exit criteria: python scripts/gen_synthetic.py --n 1000 -> 1000 samples, schema-OK,
all 6 categories balanced (~130-190 each).
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
"""Synthetic sample-composition pipeline (T1.5).
|
||||
|
||||
Composes :mod:`tsmm.data` primitives into one fully-formed, JSON-serializable
|
||||
sample::
|
||||
|
||||
generate_series ─┐
|
||||
sample_attributes ─┼─► gen_one_sample() ─► JSONL via write_jsonl()
|
||||
inject_anomaly ─┤
|
||||
couple_event ─┤
|
||||
build_instruction ─┘
|
||||
|
||||
The heavy ``scripts/gen_synthetic.py`` entrypoint is a thin multiprocessing
|
||||
wrapper around these two functions so the core logic stays unit-testable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
from typing import Any, Iterable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .attributes import sample_attributes
|
||||
from .instruct import INSTRUCTION_CATEGORIES, build_instruction
|
||||
from .synthesis import add_missing, couple_event, generate_series, inject_anomaly
|
||||
|
||||
# Default per-sample composition knobs; overridable by callers / the CLI.
|
||||
_DEFAULTS = dict(
|
||||
T=512,
|
||||
C=5,
|
||||
anomaly_types=("spike", "level_shift", "variance_change", "drift"),
|
||||
missing_rate=0.05,
|
||||
event_prob=0.5,
|
||||
)
|
||||
|
||||
|
||||
def _to_jsonable(obj: Any) -> Any:
|
||||
"""Recursively convert numpy types/arrays to plain JSON-serializable types."""
|
||||
if isinstance(obj, dict):
|
||||
return {k: _to_jsonable(v) for k, v in obj.items()}
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [_to_jsonable(v) for v in obj]
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
if isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
if isinstance(obj, (np.floating,)):
|
||||
v = float(obj)
|
||||
return None if np.isnan(v) else v
|
||||
if isinstance(obj, float):
|
||||
return None if np.isnan(obj) else obj
|
||||
return obj
|
||||
|
||||
|
||||
def gen_one_sample(
|
||||
*,
|
||||
T: int = 512,
|
||||
C: int = 5,
|
||||
seed: int | None = None,
|
||||
anomaly_types: Iterable[str] = _DEFAULTS["anomaly_types"],
|
||||
missing_rate: float = _DEFAULTS["missing_rate"],
|
||||
event_prob: float = _DEFAULTS["event_prob"],
|
||||
force_category: str | None = None,
|
||||
horizon: int = 10,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate one fully-composed synthetic sample.
|
||||
|
||||
Returns a JSON-serializable dict with keys: ``series``, ``attributes``,
|
||||
``timestamps``, ``events``, ``instruction``, ``answer``, ``labels``,
|
||||
``category`` (plus ``segments`` for downstream use).
|
||||
"""
|
||||
rng = random.Random(seed)
|
||||
|
||||
# Derive independent sub-seeds so each stochastic primitive is reproducible
|
||||
# and uncorrelated with the others.
|
||||
s_attr, s_gen, s_anom, s_event, s_miss, s_cat, s_inst = (
|
||||
rng.randrange(2**31) for _ in range(7)
|
||||
)
|
||||
|
||||
# 1) attributes + series
|
||||
attrs = sample_attributes(C, seed=s_attr)
|
||||
series = generate_series(T=T, C=C, seed=s_gen)
|
||||
|
||||
# 2) anomalies
|
||||
types = list(anomaly_types)
|
||||
n_anom = rng.randint(1, max(1, len(types)))
|
||||
chosen_types = rng.sample(types, n_anom)
|
||||
series, labels, segments = inject_anomaly(
|
||||
series, types=chosen_types, seed=s_anom
|
||||
)
|
||||
|
||||
# 3) optional event coupling (may or may not happen)
|
||||
events: list[dict[str, Any]] = []
|
||||
if rng.random() < event_prob:
|
||||
t_event = int(rng.integers(0, T)) if hasattr(rng, "integers") else int(rng.randrange(0, T))
|
||||
kind = rng.choice(["deploy", "rollback", "incident", "config", "scale"])
|
||||
series, text = couple_event(series, t=t_event, kind=kind, seed=s_event)
|
||||
events.append({"t": int(t_event), "text": text})
|
||||
|
||||
# 4) missing values
|
||||
if missing_rate > 0:
|
||||
series = add_missing(series, rate=missing_rate, seed=s_miss)
|
||||
|
||||
# 5) timestamps (synthetic unix-ish index based on the first channel's freq)
|
||||
freq_seconds = _freq_to_seconds(attrs[0]["freq"])
|
||||
t0 = 1_700_000_000
|
||||
timestamps = [int(t0 + i * freq_seconds) for i in range(T)]
|
||||
|
||||
# 6) instruction category + answer
|
||||
category = force_category if force_category is not None else random.Random(s_cat).choice(
|
||||
INSTRUCTION_CATEGORIES
|
||||
)
|
||||
instruction, answer = build_instruction(
|
||||
category=category,
|
||||
series=series,
|
||||
series_meta=attrs,
|
||||
labels=labels,
|
||||
segments=segments,
|
||||
events=events,
|
||||
seed=s_inst,
|
||||
horizon=horizon,
|
||||
)
|
||||
|
||||
sample = {
|
||||
"series": series,
|
||||
"attributes": attrs,
|
||||
"timestamps": timestamps,
|
||||
"events": events,
|
||||
"instruction": instruction,
|
||||
"answer": answer,
|
||||
"labels": labels,
|
||||
"category": category,
|
||||
"segments": segments,
|
||||
}
|
||||
return _to_jsonable(sample)
|
||||
|
||||
|
||||
def _freq_to_seconds(freq: str) -> int:
|
||||
"""Convert a frequency band like ``'1s'``/``'5min'`` to seconds."""
|
||||
import re
|
||||
|
||||
m = re.fullmatch(r"(\d+)\s*(s|min|ms|h)", freq.strip())
|
||||
if not m:
|
||||
return 1
|
||||
n, unit = int(m.group(1)), m.group(2)
|
||||
mult = {"ms": 0, "s": 1, "min": 60, "h": 3600}[unit]
|
||||
return n * mult
|
||||
|
||||
|
||||
def write_jsonl(samples: Iterable[dict[str, Any]], path: str) -> int:
|
||||
"""Write an iterable of samples to ``path`` as JSONL. Returns line count."""
|
||||
n = 0
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
for s in samples:
|
||||
f.write(json.dumps(_to_jsonable(s), ensure_ascii=False))
|
||||
f.write("\n")
|
||||
n += 1
|
||||
return n
|
||||
@@ -72,7 +72,8 @@ def _channel_names(series_meta: list[dict[str, Any]]) -> str:
|
||||
def _format_instruction(category: str, series_meta, *, horizon=10, event=None) -> str:
|
||||
tmpl = _INSTRUCTION_TEMPLATES[category]
|
||||
names = _channel_names(series_meta)
|
||||
fields = {"names": names, "horizon": horizon}
|
||||
# All templates only use a subset of these; missing keys are simply unused.
|
||||
fields = {"names": names, "horizon": horizon, "et": "?"}
|
||||
if event is not None:
|
||||
fields["et"] = event["t"]
|
||||
return tmpl[0].format(**fields)
|
||||
|
||||
Reference in New Issue
Block a user