feat: Enhance Optuna integration and logging for backtesting framework

Changes:
- Added new API endpoints for continuing and confirming Optuna jobs, allowing for better management of ongoing studies.
- Introduced detailed logging for tick feed tracking and order book processing, improving traceability of vendor performance during backtests.
- Updated database schema to include new fields for managing Optuna study results, enhancing the ability to track study progress and outcomes.
- Refactored existing functions to utilize the new logging and tracking features, ensuring consistency across the backtesting framework.

Impact:
- These enhancements improve the robustness and transparency of the Optuna backtesting process, facilitating better analysis and optimization of trading strategies.

kis_trader/ws/candle_series.py

@@ -91,6 +91,55 @@ def live_read_label() -> str:
     return f"LIVE({main})+{aux}+ls+kiwoom_rest"
 
 
+def _index_ticks_by_minute(ticks: Sequence[Dict[str, Any]]) -> Dict[Tuple[str, str], List[Dict[str, Any]]]:
+    """(source, YYYYMMDDHHMM) → 틱. source 빈 틱은 ('*', 분) — 모든 증권사 쓰레기검사에 포함."""
+    idx: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
+    for t in ticks or []:
+        if not isinstance(t, dict):
+            continue
+        raw = str(t.get("tick_time_raw") or t.get("tick_time") or "").strip()
+        digits = raw.replace(":", "").replace("-", "").replace(" ", "")
+        tmin = digits[:12] if len(digits) >= 12 else ""
+        if len(tmin) < 12 or not tmin.isdigit():
+            continue
+        tsrc = str(t.get("source") or "").strip().lower()
+        key = (tsrc if tsrc else "*", tmin)
+        idx.setdefault(key, []).append(t)
+    return idx
+
+
+def _ticks_in_bar_from_index(
+    idx: Optional[Dict[Tuple[str, str], List[Dict[str, Any]]]],
+    source: str,
+    candle_time: str,
+    tf_min: int,
+) -> List[Dict[str, Any]]:
+    """tick_in_bar_bucket 과 같은 분 구간만 — bar_is_garbage 전체 스캔 방지."""
+    from kis_trader.engine.candle_rollup import add_candle_minutes
+
+    if not idx:
+        return []
+    ct = str(candle_time or "").strip()[:12]
+    if len(ct) < 12:
+        return []
+    end = add_candle_minutes(ct, max(1, int(tf_min or 1)))
+    if not end:
+        return []
+    end12 = end[:12]
+    src = str(source or "").strip().lower()
+    out: List[Dict[str, Any]] = []
+    cur = ct
+    while cur and cur < end12:
+        if src:
+            out.extend(idx.get((src, cur), ()))
+        out.extend(idx.get(("*", cur), ()))
+        nxt = add_candle_minutes(cur, 1)
+        if not nxt or nxt[:12] <= cur:
+            break
+        cur = nxt[:12]
+    return out
+
+
 def dedupe_by_read_pairs(
     rows: Sequence[Dict[str, Any]],
     pairs: Optional[Sequence[ReadPair]] = None,
@@ -99,11 +148,13 @@ def dedupe_by_read_pairs(
     tf_min: int = 1,
     missing_policy: str = "hole",
     live_cover: bool = False,
+    stats: Optional[Dict[str, Any]] = None,
 ) -> List[Dict[str, Any]]:
     """candle_time 1행 — pairs 앞쪽이 이김. source/channel 은 결과에 남기지 않음.
 
     ticks 가 있으면 WS 봉만 쓰레기(0건·봉끝 나이초과)일 때 다음 쌍. 한 봉 혼합 없음.
     live_cover=True 이면 링이 그 분을 커버 못할 때 그 WS 봉을 유지.
+    stats 가 있으면 pick/garbage/hole 누적 (옵투나 로그용).
     """
     from kis_trader.engine.feed_fallback import bar_is_garbage, live_bar_is_garbage
 
@@ -123,30 +174,59 @@ def dedupe_by_read_pairs(
             by_ct[ct][pair] = dict(row)
 
     tf = max(1, int(tf_min or 1))
+    tick_idx = None
+    if ticks is not None:
+        tick_idx = _index_ticks_by_minute(ticks)
+    if stats is not None:
+        stats.setdefault("slots", 0)
+        stats.setdefault("picked", 0)
+        stats.setdefault("hole", 0)
+        stats.setdefault("garbage_skip", 0)
+        stats.setdefault("pick_by", {})
+        stats.setdefault("garbage_by", {})
+        stats.setdefault("raw_by", {})
+        for ct_map in by_ct.values():
+            for pair in ct_map:
+                key = "%s/%s" % pair
+                stats["raw_by"][key] = int(stats["raw_by"].get(key) or 0) + 1
     out: List[Dict[str, Any]] = []
     for ct in sorted(by_ct.keys()):
+        if stats is not None:
+            stats["slots"] = int(stats.get("slots") or 0) + 1
         picked: Optional[Dict[str, Any]] = None
+        pick_key = ""
         for pair in order:
             bar = by_ct[ct].get(pair)
             if bar is None:
                 continue
             src, ch = pair
             if ticks is not None and ch == CHANNEL_WS:
+                scoped = _ticks_in_bar_from_index(tick_idx, src, ct, tf)
                 if live_cover:
                     bad = live_bar_is_garbage(
                         ticks, candle_time=ct, tf_min=tf, source=src,
                     )
                 else:
                     bad = bar_is_garbage(
-                        ticks, candle_time=ct, tf_min=tf, source=src,
+                        scoped, candle_time=ct, tf_min=tf, source=src,
                         missing_policy=missing_policy,
                     )
                 if bad:
+                    if stats is not None:
+                        stats["garbage_skip"] = int(stats.get("garbage_skip") or 0) + 1
+                        gk = "%s/%s" % pair
+                        stats["garbage_by"][gk] = int(stats["garbage_by"].get(gk) or 0) + 1
                     continue
             picked = dict(bar)
+            pick_key = "%s/%s" % pair
             break
         if picked is None:
+            if stats is not None:
+                stats["hole"] = int(stats.get("hole") or 0) + 1
             continue
+        if stats is not None:
+            stats["picked"] = int(stats.get("picked") or 0) + 1
+            stats["pick_by"][pick_key] = int(stats["pick_by"].get(pick_key) or 0) + 1
         picked.pop("source", None)
         picked.pop("channel", None)
         out.append(picked)