"""E2E 编排器:inject → 6 算法 → FORECAST → 可视化 → JSON 日志。""" import json import logging import time from pathlib import Path from datetime import datetime from ts_anomaly_td.connector import TDConnection from ts_anomaly_td.schema import setup_schema from ts_anomaly_td.io_csv import read_csv from ts_anomaly_td.detection import detect_all_algos, ALL_ALGOS from ts_anomaly_td.forecast import forecast_anomaly from ts_anomaly_td.visualize import render_contrast, _compute_iou logger = logging.getLogger(__name__) def run_e2e(args): """执行 E2E 全流程。 Args: args: argparse namespace(含 data_dir, output_dir, log_dir, url) """ logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", ) data_dir = Path(args.data_dir) output_dir = Path(args.output_dir) log_dir = Path(args.log_dir) output_dir.mkdir(parents=True, exist_ok=True) log_dir.mkdir(parents=True, exist_ok=True) csv_files = sorted(data_dir.glob("*.csv")) if not csv_files: logger.error("no CSV files found in %s", data_dir) return 1 conn = TDConnection(args.url) summary = {"run_at": datetime.now().isoformat(), "datasets": {}, "exit_code": 0} last_log_path = None try: for csv_path in csv_files: stable = csv_path.stem logger.info("=== E2E %s ===", stable) ts, vals, labels = read_csv(str(csv_path)) rows = list(zip(ts, vals, labels)) t0 = time.monotonic() setup_schema(conn, stable) conn.batch_insert(stable, rows) inject_time = time.monotonic() - t0 logger.info("inject: %d rows in %.1fs", len(rows), inject_time) t0 = time.monotonic() det_results = detect_all_algos(conn, stable) detect_time = time.monotonic() - t0 for algo, r in det_results.items(): if r["error"]: logger.info(" %s: ERROR %s", algo, r["error"]) else: logger.info(" %s: %d windows", algo, len(r["windows"])) t0 = time.monotonic() try: fc_points = forecast_anomaly(conn, stable) fc_time = time.monotonic() - t0 fc_anom_count = sum(1 for p in fc_points if p["is_anomaly"]) logger.info("FORECAST: %d points, %d anomalies in %.1fs", len(fc_points), fc_anom_count, fc_time) except Exception as e: logger.warning("FORECAST failed (non-fatal): %s", e) fc_points = [] fc_time = time.monotonic() - t0 fc_anom_count = 0 t0 = time.monotonic() png_path = output_dir / f"{stable}_gt_vs_tdengine.png" render_contrast(ts, vals, labels, det_results, str(png_path), title=stable) viz_time = time.monotonic() - t0 ds_summary = { "rows": len(rows), "inject_time_s": round(inject_time, 1), "detect_time_s": round(detect_time, 1), "forecast_time_s": round(fc_time, 1), "viz_time_s": round(viz_time, 1), "algorithms": {}, "forecast": { "point_count": len(fc_points), "anomaly_count": fc_anom_count, }, "png": str(png_path), "iou": None, } # 计算 GT windows(从 labels 提取) gt_windows_for_iou = [] in_anom = False anom_start = None for i, l in enumerate(labels): if l == 1 and not in_anom: in_anom = True anom_start = ts[i] elif l == 0 and in_anom: in_anom = False gt_windows_for_iou.append((anom_start, ts[i])) if in_anom: gt_windows_for_iou.append((anom_start, ts[-1])) # 收集检测 windows all_det_windows = [] for r in det_results.values(): if not r.get("error"): all_det_windows.extend(r.get("windows", [])) iou = _compute_iou(gt_windows_for_iou, all_det_windows) ds_summary["iou"] = round(iou, 3) for algo, r in det_results.items(): ds_summary["algorithms"][algo] = { "window_count": len(r["windows"]), "error": r["error"], } summary["datasets"][stable] = ds_summary last_log_path = log_dir / f"e2e_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" last_log_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False)) logger.info("log saved: %s", last_log_path) except Exception as e: logger.exception("E2E failed: %s", e) summary["exit_code"] = 1 summary["error"] = str(e) last_log_path = log_dir / f"e2e_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" last_log_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False)) finally: conn.close() return summary["exit_code"]