张宗平
144 lines
5.1 KiB
Python
144 lines
5.1 KiB
Python
"""E2E 编排器：inject → 6 算法 → FORECAST → 可视化 → JSON 日志。"""
|
||
|
||
import json
|
||
import logging
|
||
import time
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
|
||
from ts_anomaly_td.connector import TDConnection
|
||
from ts_anomaly_td.schema import setup_schema
|
||
from ts_anomaly_td.io_csv import read_csv
|
||
from ts_anomaly_td.detection import detect_all_algos, ALL_ALGOS
|
||
from ts_anomaly_td.forecast import forecast_anomaly
|
||
from ts_anomaly_td.visualize import render_contrast, _compute_iou
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def run_e2e(args):
|
||
"""执行 E2E 全流程。
|
||
|
||
Args:
|
||
args: argparse namespace（含 data_dir, output_dir, log_dir, url）
|
||
"""
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||
)
|
||
|
||
data_dir = Path(args.data_dir)
|
||
output_dir = Path(args.output_dir)
|
||
log_dir = Path(args.log_dir)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
log_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
csv_files = sorted(data_dir.glob("*.csv"))
|
||
if not csv_files:
|
||
logger.error("no CSV files found in %s", data_dir)
|
||
return 1
|
||
|
||
conn = TDConnection(args.url)
|
||
summary = {"run_at": datetime.now().isoformat(), "datasets": {}, "exit_code": 0}
|
||
last_log_path = None
|
||
|
||
try:
|
||
for csv_path in csv_files:
|
||
stable = csv_path.stem
|
||
logger.info("=== E2E %s ===", stable)
|
||
|
||
ts, vals, labels = read_csv(str(csv_path))
|
||
rows = list(zip(ts, vals, labels))
|
||
|
||
t0 = time.monotonic()
|
||
setup_schema(conn, stable)
|
||
conn.batch_insert(stable, rows)
|
||
inject_time = time.monotonic() - t0
|
||
logger.info("inject: %d rows in %.1fs", len(rows), inject_time)
|
||
|
||
t0 = time.monotonic()
|
||
det_results = detect_all_algos(conn, stable)
|
||
detect_time = time.monotonic() - t0
|
||
for algo, r in det_results.items():
|
||
if r["error"]:
|
||
logger.info(" %s: ERROR %s", algo, r["error"])
|
||
else:
|
||
logger.info(" %s: %d windows", algo, len(r["windows"]))
|
||
|
||
t0 = time.monotonic()
|
||
try:
|
||
fc_points = forecast_anomaly(conn, stable)
|
||
fc_time = time.monotonic() - t0
|
||
fc_anom_count = sum(1 for p in fc_points if p["is_anomaly"])
|
||
logger.info("FORECAST: %d points, %d anomalies in %.1fs", len(fc_points), fc_anom_count, fc_time)
|
||
except Exception as e:
|
||
logger.warning("FORECAST failed (non-fatal): %s", e)
|
||
fc_points = []
|
||
fc_time = time.monotonic() - t0
|
||
fc_anom_count = 0
|
||
|
||
t0 = time.monotonic()
|
||
png_path = output_dir / f"{stable}_gt_vs_tdengine.png"
|
||
render_contrast(ts, vals, labels, det_results, str(png_path), title=stable)
|
||
viz_time = time.monotonic() - t0
|
||
|
||
ds_summary = {
|
||
"rows": len(rows),
|
||
"inject_time_s": round(inject_time, 1),
|
||
"detect_time_s": round(detect_time, 1),
|
||
"forecast_time_s": round(fc_time, 1),
|
||
"viz_time_s": round(viz_time, 1),
|
||
"algorithms": {},
|
||
"forecast": {
|
||
"point_count": len(fc_points),
|
||
"anomaly_count": fc_anom_count,
|
||
},
|
||
"png": str(png_path),
|
||
"iou": None,
|
||
}
|
||
|
||
# 计算 GT windows（从 labels 提取）
|
||
gt_windows_for_iou = []
|
||
in_anom = False
|
||
anom_start = None
|
||
for i, l in enumerate(labels):
|
||
if l == 1 and not in_anom:
|
||
in_anom = True
|
||
anom_start = ts[i]
|
||
elif l == 0 and in_anom:
|
||
in_anom = False
|
||
gt_windows_for_iou.append((anom_start, ts[i]))
|
||
if in_anom:
|
||
gt_windows_for_iou.append((anom_start, ts[-1]))
|
||
|
||
# 收集检测 windows
|
||
all_det_windows = []
|
||
for r in det_results.values():
|
||
if not r.get("error"):
|
||
all_det_windows.extend(r.get("windows", []))
|
||
|
||
iou = _compute_iou(gt_windows_for_iou, all_det_windows)
|
||
ds_summary["iou"] = round(iou, 3)
|
||
for algo, r in det_results.items():
|
||
ds_summary["algorithms"][algo] = {
|
||
"window_count": len(r["windows"]),
|
||
"error": r["error"],
|
||
}
|
||
|
||
summary["datasets"][stable] = ds_summary
|
||
|
||
last_log_path = log_dir / f"e2e_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||
last_log_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False))
|
||
logger.info("log saved: %s", last_log_path)
|
||
|
||
except Exception as e:
|
||
logger.exception("E2E failed: %s", e)
|
||
summary["exit_code"] = 1
|
||
summary["error"] = str(e)
|
||
last_log_path = log_dir / f"e2e_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||
last_log_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False))
|
||
finally:
|
||
conn.close()
|
||
|
||
return summary["exit_code"]
|