refactor: enhance Optuna backtesting framework, optimize orderbook filtering, and update database management utilities.

2026-08-12 10:19:19 +09:00
parent cb7e5037a0
commit c6bd62a25f
218 changed files with 31613 additions and 759 deletions
--- a/kis_trader/backtest/trigger_snapshot_loader.py
+++ b/kis_trader/backtest/trigger_snapshot_loader.py
@@ -20,6 +20,32 @@ LOG_BACKFILL_SOURCE = "log_backfill"
 # 키움 실시간 호가 주기 스냅샷(본체 levels_json 보유) — 스프레드 등 재계산용.
 # log_backfill(판정만)·filter_eval(판정시점만)과 달리 종목·시각을 폭넓게 커버한다.
 KIWOOM_BODY_SOURCE = "kiwoom_0d"
+# LS UH1 등 — history_source=ls 일 때 ls_ws_orderbook 본체
+LS_BODY_SOURCES = ("ls_uh1", "ls_h1", "ls_ha", "ls_nh1")
+
+
+def resolve_orderbook_history_source(
+    engine_params: Optional[Dict[str, Any]] = None,
+) -> str:
+    """봉/틱과 동일 축 — universe history_source (ls|kiwoom)."""
+    try:
+        from kis_trader.backtest.universe_history_source import (
+            normalize_universe_history_source,
+        )
+    except Exception:
+        def normalize_universe_history_source(raw):  # type: ignore
+            s = str(raw or "").strip().lower()
+            return "ls" if s in ("ls", "ls_condition", "ls_afr", "ls_ws") else "kiwoom"
+
+    if not engine_params:
+        return "kiwoom"
+    raw = (
+        engine_params.get("_orderbook_history_source")
+        or engine_params.get("_universe_history_source")
+        or engine_params.get("universe_history_source")
+        or engine_params.get("history_source")
+    )
+    return normalize_universe_history_source(raw)


 def backtest_use_kiwoom_body_snapshot(
@@ -71,7 +97,16 @@ def backtest_needs_trigger_snapshot_load(
    p = params or {}
    ob_on = _orderbook_filter_enabled_for_entry(p, strategy)
    pg_on = _program_filter_enabled_for_entry(p, strategy)
-    return bool(ob_on or pg_on)
+    # 수익구간·손절호가 ON 이면 진입필터 OFF여도 OR 본체가 필요 (모멘텀 키움 / 돌파 LS)
+    try:
+        from kis_trader.engine.momentum_hts_logic import need_ob_or_history
+
+        exit_ob_on = need_ob_or_history(p)
+    except Exception:
+        exit_ob_on = bool(
+            p.get("exit_ob_enabled", False) or p.get("stop_ob_enabled", False)
+        )
+    return bool(ob_on or pg_on or exit_ob_on)


 def backtest_use_trigger_snapshot_db(
@@ -203,8 +238,10 @@ def _load_orderbook_rows(
    market: str,
    sources: Tuple[str, ...],
    strategy: str = "",
+    table: str = "ws_orderbook",
+    use_strategy_col: bool = True,
 ) -> List[Dict[str, Any]]:
-    """ws_orderbook 행 조회 (source·strategy 필터)."""
+    """호가 스냅 행 조회 (ws_orderbook | ls_ws_orderbook)."""
    tt_start = (start_key or "")[:14]
    if len(tt_start) == 12:
        tt_start += "00"
@@ -212,6 +249,17 @@ def _load_orderbook_rows(
    if len(tt_end) == 12:
        tt_end += "59"

+    tbl = "ls_ws_orderbook" if str(table).strip().lower() == "ls_ws_orderbook" else "ws_orderbook"
+    import os
+    ob_source = os.environ.get("OB_SOURCE", "").strip()
+    if ob_source == "kis":
+        tbl = "kis_ws_orderbook"
+        sources = ()
+        use_strategy_col = False
+    elif ob_source == "kiwoom_0d":
+        tbl = "ws_orderbook"
+        sources = ("kiwoom_0d",)
+
    code_filter = ""
    params: List[Any] = [market, tt_start, tt_end]
    if codes:
@@ -224,19 +272,31 @@ def _load_orderbook_rows(
        source_filter = f" AND source IN ({','.join(['%s'] * len(sources))})"
        params.extend(sources)

-    strat_vals = _strategy_db_values(strategy)
    strat_filter = ""
-    if strat_vals:
-        strat_filter = (
-            f" AND (strategy IS NULL OR strategy IN ({','.join(['%s'] * len(strat_vals))}))"
+    if use_strategy_col and tbl == "ws_orderbook":
+        strat_vals = _strategy_db_values(strategy)
+        if strat_vals:
+            strat_filter = (
+                f" AND (strategy IS NULL OR strategy IN ({','.join(['%s'] * len(strat_vals))}))"
+            )
+            params.extend(strat_vals)
+
+    # ls 테이블에는 strategy/reject 컬럼 없음 — 본체만
+    if tbl == "ls_ws_orderbook":
+        cols = (
+            "code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty, "
+            "bid_qty_l3, ask_qty_l3, levels_json, source, recv_ts"
+        )
+    else:
+        cols = (
+            "code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty, "
+            "bid_qty_l3, ask_qty_l3, levels_json, source, recv_ts, "
+            "strategy, reject_code, reject_msg"
        )
-        params.extend(strat_vals)

    sql = f"""
-        SELECT code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty,
-               bid_qty_l3, ask_qty_l3, levels_json, source, recv_ts,
-               strategy, reject_code, reject_msg
-        FROM ws_orderbook
+        SELECT {cols}
+        FROM {tbl}
        WHERE market = %s
          AND snap_time >= %s
          AND snap_time <= %s
@@ -249,7 +309,7 @@ def _load_orderbook_rows(
        rows = db.conn.execute(sql, tuple(params)).fetchall()
        return [dict(r) for r in rows]
    except Exception as e:
-        logger.warning("ws_orderbook 조회 실패 — TRIGGER 호가 스킵: %s", e)
+        logger.warning("%s 조회 실패 — TRIGGER 호가 스킵: %s", tbl, e)
        return []


@@ -282,8 +342,50 @@ def load_orderbook_snapshots_by_code(
    engine_params: Optional[Dict[str, Any]] = None,
    strategy: str = "",
 ) -> Tuple[Dict[str, Dict[str, List[OrderbookSnapshot]]], int, Dict[str, Dict[str, List[Dict[str, Any]]]]]:
-    """기간 내 ``ws_orderbook`` → ``{code: {minute: [snap, ...]}}`` + log_backfill 판정."""
+    """기간 내 호가 → ``{code: {minute: [snap, ...]}}``.
+
+    history_source=ls → ``ls_ws_orderbook`` 본체만 (키움 log_backfill/filter_eval 폴백 금지).
+    """
    mkt = (market or get_env_from_db("WS_TICK_DEFAULT_MARKET", "KR") or "KR").strip().upper()
+    hist_src = resolve_orderbook_history_source(engine_params)
+
+    # ── LS: 본체만, 판정 메모 폴백 없음 ──
+    if hist_src == "ls":
+        try:
+            if hasattr(db, "ensure_ws_orderbook_table"):
+                # ls 테이블은 TradeDB migrate 에서 생성됨 — no-op ensure 없으면 무시
+                pass
+        except Exception:
+            pass
+        rows = _load_orderbook_rows(
+            db, start_key, end_key, codes, market=mkt,
+            sources=LS_BODY_SOURCES, strategy="",
+            table="ls_ws_orderbook", use_strategy_col=False,
+        )
+        # source 필터에 안 잡힌 구 데이터도 있으면 전체 재조회 (source 비어있는 행)
+        if not rows:
+            rows = _load_orderbook_rows(
+                db, start_key, end_key, codes, market=mkt,
+                sources=tuple(), strategy="",
+                table="ls_ws_orderbook", use_strategy_col=False,
+            )
+        out: Dict[str, Dict[str, List[OrderbookSnapshot]]] = defaultdict(dict)
+        total = 0
+        for r in rows:
+            code = str(r.get("code") or "").strip()
+            st = str(r.get("snap_time") or "").strip()
+            minute_key = _minute_key(st)
+            if not code or not minute_key:
+                continue
+            snap = orderbook_snapshot_from_storage(r)
+            bucket = out[code].setdefault(minute_key, [])
+            bucket.append(snap)
+            total += 1
+        if engine_params is not None:
+            engine_params["_orderbook_history_source"] = "ls"
+            engine_params["_backtest_disable_log_orderbook_verdict"] = True
+        return dict(out), total, {}
+
    try:
        if hasattr(db, "ensure_ws_orderbook_table"):
            db.ensure_ws_orderbook_table()
@@ -293,13 +395,25 @@ def load_orderbook_snapshots_by_code(
    use_eval = backtest_use_trigger_eval_snapshot(engine_params, strategy=strategy)
    use_log_bf = backtest_use_log_backfill_snapshot(engine_params, strategy=strategy)
    use_kiwoom = backtest_use_kiwoom_body_snapshot(engine_params, strategy=strategy)
+    # 수익구간·손절호가: 보유 중 OR_MA 용 본체 스냅 필요.
+    # 진입필터 재계산 플래그(use_kiwoom)와 분리 — 본체만 추가 적재.
+    need_exit_ob = False
+    try:
+        from kis_trader.engine.momentum_hts_logic import need_ob_or_history
+
+        need_exit_ob = need_ob_or_history(engine_params or {})
+    except Exception:
+        _p = engine_params or {}
+        need_exit_ob = bool(
+            _p.get("exit_ob_enabled", False) or _p.get("stop_ob_enabled", False)
+        )

    sources: List[str] = []
    if use_eval:
        sources.append(FILTER_EVAL_SOURCE)
    if use_log_bf:
        sources.append(LOG_BACKFILL_SOURCE)
-    if use_kiwoom:
+    if use_kiwoom or need_exit_ob:
        sources.append(KIWOOM_BODY_SOURCE)
    if not sources:
        return {}, 0, {}
@@ -307,9 +421,10 @@ def load_orderbook_snapshots_by_code(
    rows = _load_orderbook_rows(
        db, start_key, end_key, codes, market=mkt,
        sources=tuple(sources), strategy=strategy,
+        table="ws_orderbook", use_strategy_col=True,
    )

-    out: Dict[str, Dict[str, List[OrderbookSnapshot]]] = defaultdict(dict)
+    out = defaultdict(dict)
    log_verdict: Dict[str, Dict[str, List[Dict[str, Any]]]] = defaultdict(dict)
    seen_eval: Set[Tuple[str, str]] = set()
    # 본체(kiwoom_0d) 다운샘플: (code, minute) 당 최신 1건만 유지 (분단위 백테·메모리 절약)
@@ -465,12 +580,15 @@ def load_trigger_snapshots_by_code(
    )
    meta = snapshot_coverage_stats(ob_by_code, pg_by_code)
    meta["ws_orderbook_rows_loaded"] = ob_rows
+    meta["orderbook_history_source"] = resolve_orderbook_history_source(engine_params)
    meta["ws_program_rows_loaded"] = pg_rows
    meta["trigger_eval_only"] = backtest_use_trigger_eval_snapshot(
        engine_params, strategy=strategy,
    )
-    meta["log_backfill_enabled"] = backtest_use_log_backfill_snapshot(
-        engine_params, strategy=strategy,
+    meta["log_backfill_enabled"] = (
+        False
+        if meta["orderbook_history_source"] == "ls"
+        else backtest_use_log_backfill_snapshot(engine_params, strategy=strategy)
    )
    meta["log_verdict_by_code"] = log_verdict
    meta["log_verdict_rows"] = sum(