feat: 틱/호가 데이터 출처 표출 UI 추가 및 데이터 수집 개선

- 백테스트 및 실거래 시 틱과 호가의 벤더 출처(ob_source, entry_source) 기록 및 추적 강화 (tail_engine.py)
- 웹 UI '체결디버그'에 [틱:kis / 호가:ls] 형태로 데이터 출처를 직관적으로 표출 (backtest.js, backtest.html)
- LS WebSocket 구독 100건 제한 하드코딩 해제 및 env_config_ext 연동 (ls_ws.py)
- 기타 백테스트 웹 및 DB 관련 최적화 적용

database.py

@@ -1439,6 +1439,9 @@ ENV_CONFIG_KEYS = (
     "LS_WS_ORDERBOOK_SAVE",
     # 종목당 DB INSERT 최소 간격(ms) — 호가 폭주 시 DB 부하 완화 (RAM 캐시는 매 틱)
     "LS_WS_ORDERBOOK_SAVE_MS",
+    # ls_ws_ticks / ls_ws_orderbook 보존 일수 (기본 7)
+    "LS_WS_TICK_KEEP_DAYS",
+    "LS_WS_ORDERBOOK_KEEP_DAYS",
     # LS VI(UVI) 구독·적재 — 워치독 오탐 방지·백테 구간 마킹 (프로그램·상하한가 미포함)
     "LS_WS_UVI_ENABLED",
     "LS_WS_VI_SAVE",
@@ -1743,6 +1746,10 @@ class TradeDB:
         """
         allowed = {
             "target_candidates_history": "target_candidates_history",
+            "ws_ticks": "ws_ticks",
+            "ws_orderbook": "ws_orderbook",
+            "ws_ticks_us": "ws_ticks_us",
+            "ls_ws_orderbook": "ls_ws_orderbook",
         }
         tbl = allowed.get(str(table or "").strip())
         name = str(idx_name or "").strip()
@@ -2419,11 +2426,18 @@ class TradeDB:
                     tot_volume DOUBLE NULL,
                     chetime VARCHAR(16) NULL,
                     tr_cd VARCHAR(8) NULL,
+                    tk CHAR(14) NULL,
+                    lag_sec DOUBLE NULL,
                     INDEX idx_ls_tick_ts (ts),
-                    INDEX idx_ls_tick_code_ts (code, ts)
+                    INDEX idx_ls_tick_code_ts (code, ts),
+                    INDEX idx_ls_tick_code_tk (code, tk)
                 ) CHARACTER SET utf8mb4
             """)
-            logger.info("📌 ls_ws_ticks 테이블 확인/생성")
+            # Migration: add tk, lag_sec if not exist
+            cols = [dict(r)["Field"] for r in self.conn.execute("SHOW COLUMNS FROM ls_ws_ticks").fetchall()]
+            if "tk" not in cols:
+                self.conn.execute("ALTER TABLE ls_ws_ticks ADD COLUMN tk CHAR(14) NULL, ADD COLUMN lag_sec DOUBLE NULL, ADD INDEX idx_ls_tick_code_tk (code, tk)")
+            logger.info("📌 ls_ws_ticks 테이블 확인/생성 및 마이그레이션 완료")
         except Exception as e:
             logger.warning(f"migrate ls_ws_ticks 실패: {e}")
         try:
@@ -2462,12 +2476,19 @@ class TradeDB:
                     levels_json MEDIUMTEXT,
                     source VARCHAR(16) NOT NULL DEFAULT 'ls_uh1',
                     recv_ts VARCHAR(30) NOT NULL,
+                    tk CHAR(14) NULL,
+                    lag_sec DOUBLE NULL,
                     KEY idx_ls_ob_lookup (market, code, snap_time),
                     KEY idx_ls_ob_recv (recv_ts),
-                    KEY idx_ls_ob_code (code)
+                    KEY idx_ls_ob_code (code),
+                    KEY idx_ls_ob_code_snap (code, snap_time)
                 ) CHARACTER SET utf8mb4
             """)
-            logger.info("📌 ls_ws_orderbook 테이블 확인/생성")
+            # Migration: add tk, lag_sec if not exist
+            cols = [dict(r)["Field"] for r in self.conn.execute("SHOW COLUMNS FROM ls_ws_orderbook").fetchall()]
+            if "tk" not in cols:
+                self.conn.execute("ALTER TABLE ls_ws_orderbook ADD COLUMN tk CHAR(14) NULL, ADD COLUMN lag_sec DOUBLE NULL, ADD INDEX idx_ls_ob_code_snap (code, snap_time)")
+            logger.info("📌 ls_ws_orderbook 테이블 확인/생성 및 마이그레이션 완료")
         except Exception as e:
             logger.warning(f"migrate ls_ws_orderbook 실패: {e}")
         try:
@@ -2564,6 +2585,21 @@ class TradeDB:
             "idx_tch_slot_sid",
             "(slot_key, strategy_id)",
         )
+        self._migrate_feed_collect_stats_indexes()
+
+    def _migrate_feed_collect_stats_indexes(self) -> None:
+        """수집통계·일자 범위 조회 — tick_time/snap_time 단독 인덱스 (타입 변경 없음)."""
+        specs = (
+            ("ws_ticks", "idx_ws_ticks_tick_time", "(tick_time)"),
+            ("ws_ticks", "idx_ws_ticks_day_src", "(tick_time, source)"),
+            ("ws_ticks", "idx_ws_ticks_src_code_recv", "(source, code, recv_ts)"),
+            ("ws_orderbook", "idx_ws_ob_snap", "(snap_time)"),
+            ("ws_orderbook", "idx_ws_ob_snap_src", "(snap_time, source)"),
+            ("ws_ticks_us", "idx_ws_ticks_us_tick_time", "(tick_time)"),
+            ("ls_ws_orderbook", "idx_ls_ob_snap", "(snap_time)"),
+        )
+        for tbl, name, cols in specs:
+            self._ensure_safe_index(tbl, name, cols)
 
     def _migrate_ws_candles_source_channel(self) -> None:
         """ws_candles.channel 추가 + UNIQUE(code,tf,time,source,channel) + rest 라벨 정규화."""
@@ -3945,13 +3981,15 @@ class TradeDB:
         tot_volume: Optional[float] = None,
         chetime: str = "",
         tr_cd: str = "",
+        tk: Optional[str] = None,
+        lag_sec: Optional[float] = None,
     ) -> bool:
         try:
             self.conn.execute(
                 "INSERT INTO ls_ws_ticks "
-                "(ts, code, price, volume, tot_volume, chetime, tr_cd) "
-                "VALUES (%s,%s,%s,%s,%s,%s,%s)",
-                (ts, code, price, volume, tot_volume, chetime or None, tr_cd or None),
+                "(ts, code, price, volume, tot_volume, chetime, tr_cd, tk, lag_sec) "
+                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+                (ts, code, price, volume, tot_volume, chetime or None, tr_cd or None, tk, lag_sec),
             )
             return True
         except Exception as e:
@@ -4129,6 +4167,8 @@ class TradeDB:
         code: str,
         snap: Dict[str, Any],
         market: str = "KR",
+        tk: Optional[str] = None,
+        lag_sec: Optional[float] = None,
     ) -> bool:
         """LS UH1 호가 스냅샷 → ls_ws_orderbook (키움 ws_orderbook 스키마 대칭)."""
         try:
@@ -4140,8 +4180,8 @@ class TradeDB:
                 "INSERT INTO ls_ws_orderbook "
                 "(market, code, snap_time, best_bid, best_ask, "
                 "total_bid_qty, total_ask_qty, bid_qty_l3, ask_qty_l3, "
-                "levels_json, source, recv_ts) "
-                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+                "levels_json, source, recv_ts, tk, lag_sec) "
+                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
                 (
                     (market or "KR")[:8],
                     code,
@@ -4155,6 +4195,8 @@ class TradeDB:
                     snap.get("levels_json"),
                     str(snap.get("source") or "ls_uh1")[:16],
                     recv_ts,
+                    tk,
+                    lag_sec,
                 ),
             )
             return True
@@ -5142,17 +5184,94 @@ class TradeDB:
             logger.error("get_latest_confirmed_ws_candle 실패(%s): %s", code, e)
             return None
 
-    def cleanup_old_ws_candles(self, keep_days: int = 3):
-        """오래된 ws_candles 정리 (기본 3일 이상 지난 봉 삭제)."""
+    def cleanup_old_ws_candles(self, keep_days: int = 7) -> None:
+        """오래된 ws_candles 정리 (기본 7일 이상 지난 봉 삭제, 청크 방식)."""
+        if keep_days <= 0:
+            return
         cutoff = (datetime.datetime.now() - datetime.timedelta(days=keep_days)).strftime("%Y%m%d%H%M")
+        chunk = max(1000, int(os.environ.get("WS_CANDLE_CLEANUP_CHUNK", "5000") or 5000))
+        max_loops = max(1, int(os.environ.get("WS_CANDLE_CLEANUP_MAX_LOOPS", "200") or 200))
+        total = 0
         try:
-            with self.conn:
-                self.conn.execute(
-                    "DELETE FROM ws_candles WHERE candle_time < ?", (cutoff,)
+            import time as _time
+            for _ in range(max_loops):
+                cur = self.conn.execute(
+                    "DELETE FROM ws_candles WHERE candle_time < %s LIMIT %s",
+                    (cutoff, chunk),
+                )
+                n = int(getattr(cur, "rowcount", 0) or 0)
+                total += n
+                if n < chunk:
+                    break
+                _time.sleep(0.05)
+            if total:
+                logger.info(
+                    "🧹 ws_candles 정리 %d행 (candle_time < %s, chunk=%d)",
+                    total, cutoff, chunk,
                 )
         except Exception as e:
             logger.error("cleanup_old_ws_candles 실패: %s", e)
 
+    def cleanup_old_ls_ws_ticks(self, keep_days: int = 7) -> None:
+        """ls_ws_ticks 오래된 틱 정리 (ts 컬럼 기준, 청크 방식)."""
+        if keep_days <= 0:
+            return
+        cutoff = (
+            datetime.datetime.now() - datetime.timedelta(days=keep_days)
+        ).strftime("%Y-%m-%d %H:%M:%S")
+        chunk = max(1000, int(os.environ.get("LS_TICK_CLEANUP_CHUNK", "5000") or 5000))
+        max_loops = max(1, int(os.environ.get("LS_TICK_CLEANUP_MAX_LOOPS", "200") or 200))
+        total = 0
+        try:
+            import time as _time
+            for _ in range(max_loops):
+                cur = self.conn.execute(
+                    "DELETE FROM ls_ws_ticks WHERE ts < %s LIMIT %s",
+                    (cutoff, chunk),
+                )
+                n = int(getattr(cur, "rowcount", 0) or 0)
+                total += n
+                if n < chunk:
+                    break
+                _time.sleep(0.05)
+            if total:
+                logger.info(
+                    "🧹 ls_ws_ticks 정리 %d행 (ts < %s, chunk=%d)",
+                    total, cutoff, chunk,
+                )
+        except Exception as e:
+            logger.error("cleanup_old_ls_ws_ticks 실패: %s", e)
+
+    def cleanup_old_ls_ws_orderbook(self, keep_days: int = 7) -> None:
+        """ls_ws_orderbook 오래된 스냅샷 정리 (recv_ts 기준, 청크 방식)."""
+        if keep_days <= 0:
+            return
+        cutoff = (
+            datetime.datetime.now() - datetime.timedelta(days=keep_days)
+        ).strftime("%Y-%m-%d %H:%M:%S")
+        chunk = max(1000, int(os.environ.get("LS_OB_CLEANUP_CHUNK", "5000") or 5000))
+        max_loops = max(1, int(os.environ.get("LS_OB_CLEANUP_MAX_LOOPS", "200") or 200))
+        total = 0
+        try:
+            import time as _time
+            for _ in range(max_loops):
+                cur = self.conn.execute(
+                    "DELETE FROM ls_ws_orderbook WHERE recv_ts < %s LIMIT %s",
+                    (cutoff, chunk),
+                )
+                n = int(getattr(cur, "rowcount", 0) or 0)
+                total += n
+                if n < chunk:
+                    break
+                _time.sleep(0.05)
+            if total:
+                logger.info(
+                    "🧹 ls_ws_orderbook 정리 %d행 (recv_ts < %s, chunk=%d)",
+                    total, cutoff, chunk,
+                )
+        except Exception as e:
+            logger.error("cleanup_old_ls_ws_orderbook 실패: %s", e)
+
     # ==================================================================
     # ws_ticks — 실시간 체결 틱 (TickRecorder 배치 INSERT)
     # ==================================================================