#!/usr/bin/env python3 """ 모멘텀 백테스트 공통 — backtest_web / param_search 단일 진입점. 실매 MomentumStrategy 와 동일: - 진입: ``momentum_engine.check_buy_signal_momentum_live`` - 청산: ws_ticks 틱 리플레이(실매 체결가) 또는 ``check_sell_signal_momentum_backtest_bar`` 폴백 """ from __future__ import annotations import time from typing import Any, Dict, List, Optional, Set, Tuple from kis_trader.backtest.backtest_portfolio_common import ( attach_scalp_trade_pnl, backtest_slip_pct, build_budget_warning, fee_and_slot_from_env_row, merge_portfolio_into_params, resolve_portfolio_params, resolve_trigger_snapshots_for_backtest, summarize_trades, ) from kis_trader.engine.momentum_engine import ( MOMENTUM_STRATEGY_ID, get_momentum_defaults_from_db, run_momentum_backtest, ) from kis_trader.engine.indicator_cache import ( materialize_ws_candles_batch, ws_candles_select_indicator_cols, ) MOMENTUM_STRATEGY = MOMENTUM_STRATEGY_ID def momentum_backtest_universe_strict_enabled() -> bool: """모멘텀 백테: 종목별 첫 event_time 이후 분만 유니버스 (실매 정합).""" from kis_trader.utils.env import get_env_bool return get_env_bool("MOMENTUM_BACKTEST_UNIVERSE_STRICT", True) def momentum_backtest_universe_strict_lag_min() -> int: """첫 편입 분 이후 추가 대기 분 (기본 1 → 09:42:25 편입은 09:43 분봉부터).""" from kis_trader.utils.env import get_env_int return max(0, int(get_env_int("MOMENTUM_BACKTEST_UNIVERSE_STRICT_LAG_MIN", 1))) def momentum_universe_exit_debounce_sec() -> int: """ 백테 EXIT 디바운스(초). 실매 ``CONDITION_EXIT_GRACE_SEC`` 와 동일하게 스냅샷 축소 시 N초 유지 (overnight→장초 wipe / 단발 EXIT 노이즈 정합). ``MOMENTUM_UNIVERSE_EXIT_DEBOUNCE_SEC`` 가 있으면 그 값 우선(0=OFF). """ from kis_trader.backtest.universe_timeline import universe_exit_debounce_sec_for_strategy return universe_exit_debounce_sec_for_strategy("MOMENTUM") def momentum_backtest_candle_warmup_bars() -> int: """백테 지표·전일종가(K) warm-up — 실매 갭보정(~500)과 맞춰 전일 세션까지 덮음.""" from kis_trader.utils.env import get_env_int return max(0, int(get_env_int("MOMENTUM_BACKTEST_CANDLE_WARMUP_BARS", 400))) def prepend_momentum_candle_warmup( db, candles_by_code: Dict[str, List[Dict]], period_start_key: str, *, warmup_bars: Optional[int] = None, ) -> int: """ ``period_start_key``(YYYYMMDDHHMM) 이전 N봉을 종목별로 prepend. RSI·EMA·패턴 판별용 — 루프 시각(all_times)에는 포함하지 않음. """ wb = ( momentum_backtest_candle_warmup_bars() if warmup_bars is None else max(0, int(warmup_bars)) ) if wb <= 0 or db is None or not period_start_key: return 0 ps = str(period_start_key)[:12] ind_cols = ws_candles_select_indicator_cols(db) total_prepended = 0 from kis_trader.backtest.bt_candle_source import fetch_ws_candles_warmup_before for code, rows in list(candles_by_code.items()): if not rows: continue first_period_idx = None for i, r in enumerate(rows): ct = str(r.get("candle_time") or "") if ct >= ps: first_period_idx = i break if first_period_idx is None: continue if first_period_idx > 0: continue first_ct = str(rows[first_period_idx].get("candle_time") or "") if not first_ct: continue warm_rows = fetch_ws_candles_warmup_before( db, code, 1, first_ct, wb, extra_select=ind_cols, ) if not warm_rows: continue prefix = warm_rows candles_by_code[code] = prefix + [dict(r) for r in rows] total_prepended += len(prefix) if total_prepended > 0: materialize_ws_candles_batch(db, candles_by_code, 1) return total_prepended # 종목×기간일 단위 REST 웜업 캐시 (프로세스 메모리만 — DB 미기록) _REST_WARMUP_PREFIX_CACHE: Dict[Tuple[str, str], List[Dict[str, Any]]] = {} # 1차+재시도 후에도 전일 종가 미확보 → trial마다 재조회·로그 금지 _REST_WARMUP_PERM_FAIL: Set[Tuple[str, str]] = set() def _momentum_rows_have_prev_day(rows: List[Dict], period_day: str) -> bool: """기간 시작일 기준 직전 거래일 **종가** 봉이 있으면 HTS K(전일종가 대비) 해석 가능. 전일 봉이 하나도 없으면 False → REST 웜업으로 보강. (시가/장시작 판정은 쓰지 않음 — HTS K 는 전일 종가 기준) """ from kis_trader.engine.momentum_hts_logic import candles_have_prev_session_close return candles_have_prev_session_close(rows or [], str(period_day or "")[:8]) def _kiwoom_gap_credentials() -> Tuple[str, str, bool]: """실매 갭보정과 동일 — REAL 우선, 없으면 LEGACY.""" from kis_trader.utils.env import get_env_from_db, get_env_bool key = (get_env_from_db("KIWOOM_APP_KEY_REAL", "") or "").strip() secret = (get_env_from_db("KIWOOM_APP_SECRET_REAL", "") or "").strip() if not key or not secret: key = (get_env_from_db("KIWOOM_APP_KEY", "") or "").strip() secret = (get_env_from_db("KIWOOM_APP_SECRET", "") or "").strip() is_mock = get_env_bool("MOMENTUM_BACKTEST_REST_KIWOOM_MOCK", False) return key, secret, is_mock def _rest_df_to_prefix( df: Any, rows: List[Dict], period_start_key: str, ) -> List[Dict[str, Any]]: """ka10080 DF → 기간 시작 이전 prefix 봉 리스트.""" ps = str(period_start_key or "")[:12] first_ct = "" for r in rows: ct = str(r.get("candle_time") or "") if ct >= ps: first_ct = ct[:12] break if not first_ct: first_ct = ps existing = {str(r.get("candle_time") or "")[:12] for r in rows} prefix: List[Dict[str, Any]] = [] for _, rec in df.iterrows(): t = str(rec.get("time") or "")[:12] if len(t) < 12 or t >= first_ct or t in existing: continue op = float(rec.get("open") or 0) if op <= 0: continue prefix.append({ "candle_time": t, "open": op, "high": float(rec.get("high") or op), "low": float(rec.get("low") or op), "close": float(rec.get("close") or op), "volume": int(float(rec.get("volume") or 0)), "is_confirmed": 1, "_rest_warmup": 1, }) prefix.sort(key=lambda x: str(x.get("candle_time") or "")) return prefix def inject_momentum_rest_warmup_memory( candles_by_code: Dict[str, List[Dict]], period_start_key: str, *, universe_by_slot: Optional[Dict[str, List[str]]] = None, ) -> Dict[str, int]: """ DB 전일봉이 없을 때 키움 ka10080 REST를 종목당 호출해 **메모리에만** prepend. - 1차: MOMENTUM_BACKTEST_REST_WARMUP_BARS (기본 700) - 전일 종가 미확보 시에만 2차: MOMENTUM_BACKTEST_REST_WARMUP_BARS_RETRY (기본 1500) - DB INSERT 없음. 성공 prefix·영구실패는 프로세스 캐시 (Optuna trial 재조회·로그 스팸 방지). """ from kis_trader.utils.env import get_env_bool, get_env_float, get_env_int from kis_trader.utils.logger import get_logger log = get_logger("kis_trader.momentum_backtest") stats = { "need": 0, "ok": 0, "fail": 0, "cache_hit": 0, "bars": 0, "skipped": 0, "retry": 0, } if not get_env_bool("MOMENTUM_BACKTEST_REST_WARMUP", True): stats["skipped"] = 1 return stats ps = str(period_start_key or "")[:12] if len(ps) < 8 or not candles_by_code: return stats period_day = ps[:8] target: Set[str] if universe_by_slot: target = set() for codes in universe_by_slot.values(): for c in codes or []: if c: target.add(str(c).strip()) target &= set(candles_by_code.keys()) else: target = set(candles_by_code.keys()) need_codes = [ c for c in sorted(target) if (c, period_day) not in _REST_WARMUP_PERM_FAIL and not _momentum_rows_have_prev_day(candles_by_code.get(c) or [], period_day) ] stats["need"] = len(need_codes) if not need_codes: return stats max_codes = int(get_env_int("MOMENTUM_BACKTEST_REST_MAX_CODES", 0)) if max_codes > 0: need_codes = need_codes[:max_codes] n_bars = max( 50, int(get_env_int("MOMENTUM_BACKTEST_REST_WARMUP_BARS", 700)), ) # 전일(직전 세션) 종가가 1차에 안 잡힐 때만 — 평소엔 700만 n_retry = max( n_bars, int(get_env_int("MOMENTUM_BACKTEST_REST_WARMUP_BARS_RETRY", 1500)), ) sleep_sec = float(get_env_float("MOMENTUM_BACKTEST_REST_SLEEP_SEC", 0.25)) kw_key, kw_secret, is_mock = _kiwoom_gap_credentials() if not kw_key or not kw_secret: log.warning("⚠️ 모멘텀 REST 웜업 스킵 — 키움 앱키/시크릿 없음") stats["fail"] = len(need_codes) return stats from kis_trader.ws.kis_ws import get_kiwoom_candles_df # 전부 캐시 hit면 Optuna trial 경로에서 INFO 스팸 금지 will_fetch = any( (c, period_day) not in _REST_WARMUP_PREFIX_CACHE for c in need_codes ) if will_fetch: log.info( "📡 모멘텀 REST 웜업(메모리): 전일봉 부족 %d종목 · ka10080 n=%d" " (실패 시 n=%d 1회 재시도, DB 미기록)", len(need_codes), n_bars, n_retry, ) for i, code in enumerate(need_codes): rows = candles_by_code.get(code) or [] if not rows: _REST_WARMUP_PERM_FAIL.add((code, period_day)) stats["fail"] += 1 continue cache_key = (code, period_day) cached = _REST_WARMUP_PREFIX_CACHE.get(cache_key) did_network = False if cached is not None: stats["cache_hit"] += 1 prefix = [dict(r) for r in cached] else: did_network = True try: df = get_kiwoom_candles_df( code, 1, kw_key, kw_secret, is_mock=is_mock, n=n_bars, ) except Exception as e: log.warning("⚠️ REST 웜업 실패 %s: %s", code, e) _REST_WARMUP_PERM_FAIL.add(cache_key) stats["fail"] += 1 continue if df is None or getattr(df, "empty", True): _REST_WARMUP_PERM_FAIL.add(cache_key) stats["fail"] += 1 continue try: prefix = _rest_df_to_prefix(df, rows, ps) except Exception as e: log.warning("⚠️ REST 웜업 파싱 실패 %s: %s", code, e) _REST_WARMUP_PERM_FAIL.add(cache_key) stats["fail"] += 1 continue # 1차로 전일 종가 미확보 → 봉 수 늘려 1회만 재시도 (중간 거래일 0봉 등) if (not prefix or not _momentum_rows_have_prev_day(prefix, period_day)) and n_retry > n_bars: stats["retry"] += 1 log.info( "📡 REST 웜업 재시도 %s: n=%d → n=%d (전일 종가 미확보)", code, n_bars, n_retry, ) try: df2 = get_kiwoom_candles_df( code, 1, kw_key, kw_secret, is_mock=is_mock, n=n_retry, ) except Exception as e: log.warning("⚠️ REST 웜업 재시도 실패 %s: %s", code, e) df2 = None if df2 is not None and not getattr(df2, "empty", True): try: prefix = _rest_df_to_prefix(df2, rows, ps) except Exception as e: log.warning("⚠️ REST 웜업 재시도 파싱 실패 %s: %s", code, e) prefix = [] _REST_WARMUP_PREFIX_CACHE[cache_key] = [dict(r) for r in prefix] if sleep_sec > 0 and did_network and i + 1 < len(need_codes): time.sleep(sleep_sec) if not prefix or not _momentum_rows_have_prev_day(prefix, period_day): _REST_WARMUP_PERM_FAIL.add(cache_key) stats["fail"] += 1 continue candles_by_code[code] = [dict(r) for r in prefix] + [dict(r) for r in rows] stats["ok"] += 1 stats["bars"] += len(prefix) if will_fetch: log.info( "✅ 모멘텀 REST 웜업 완료: ok=%d fail=%d cache=%d retry=%d bars=%d", stats["ok"], stats["fail"], stats["cache_hit"], stats["retry"], stats["bars"], ) return stats def date_keys(start: str, end: str) -> Tuple[str, str, str, str]: start_key = start.replace("-", "") + "0000" end_key = end.replace("-", "") + "2359" return start_key, end_key, start_key[:8], end_key[:8] def resolve_momentum_universe( start_ymd: str, end_ymd: str, *, use_saved_history: bool, strategy_id: str = MOMENTUM_STRATEGY_ID, history_source: str = "kiwoom", ) -> Tuple[Optional[Dict[str, List[str]]], str, int, int, str]: """ Returns: universe_by_slot, source_label, slot_count, scan_interval_min, universe_timing universe_timing: 'strict' | 'minute' | 'all' """ if use_saved_history and strategy_id: try: from kis_trader.database.db_manager import get_db as _get_ext_db from kis_trader.backtest.universe_history_source import ( history_source_label, resolve_backtest_universe_history_source, ) strict = momentum_backtest_universe_strict_enabled() lag_min = momentum_backtest_universe_strict_lag_min() debounce_sec = momentum_universe_exit_debounce_sec() hs = resolve_backtest_universe_history_source(history_source) history = _get_ext_db().get_universe_by_candle_time( strategy_id=strategy_id, start_ymd=start_ymd, end_ymd=end_ymd, strict=strict, strict_lag_minutes=lag_min, exit_debounce_sec=debounce_sec, history_source=hs, ) if history: timing = "strict" if strict else "minute" label = history_source_label(hs, strict=strict) return history, label, len(history), 1, timing except Exception: pass return None, "all", 0, 1, "all" def load_momentum_candles_by_code( db, start_key: str, end_key: str, *, warmup_bars: Optional[int] = None, market: Optional[str] = None, ) -> Tuple[Dict[str, List[Dict]], int]: """ market: None/빈값 = 전체(기존 동작), 'US'|'KR' = ws_candles.market 필터. """ from kis_trader.backtest.bt_candle_source import ( fetch_ws_candles_for_code, list_ws_candle_codes, ) period_start = str(start_key)[:12] mk = (market or "").strip().upper() codes = list_ws_candle_codes(db, 1, start_key, end_key, market=mk or None) ind_cols = ws_candles_select_indicator_cols(db) candles_by_code: Dict[str, List[Dict]] = {} total = 0 for code in codes: rows = fetch_ws_candles_for_code( db, code, 1, start_key, end_key, extra_select=ind_cols, market=mk or None, confirmed_only=False, ) if len(rows) < 6: continue candles_by_code[code] = rows total += len(rows) prepend_momentum_candle_warmup( db, candles_by_code, period_start, warmup_bars=warmup_bars, ) materialize_ws_candles_batch(db, candles_by_code, 1) return candles_by_code, total def resolve_momentum_portfolio_params( env_row: Dict[str, Any], base_params: Dict[str, Any], *, slot_money: Optional[float] = None, max_stocks: Optional[int] = None, total_budget_krw: Optional[float] = None, ) -> Dict[str, Any]: return resolve_portfolio_params( env_row, base_params, strategy="MOMENTUM", slot_money=slot_money, max_stocks=max_stocks, total_budget_krw=total_budget_krw, ) def run_momentum_backtest_web_aligned( candles_by_code: Dict[str, List[Dict]], params: Dict[str, Any], universe_by_slot: Optional[Dict[str, List[str]]] = None, *, slot_money: float = 3_000_000.0, fee_rate: float = 0.00015, sell_tax: float = 0.0018, max_stocks: int = 3, total_budget_krw: float = 0.0, ticks_by_code: Optional[Dict[str, Dict[str, List[Dict]]]] = None, orderbook_by_code: Optional[Dict[str, Dict[str, List[Any]]]] = None, program_by_code: Optional[Dict[str, Dict[str, List[Any]]]] = None, meta_out: Optional[Dict[str, Any]] = None, ) -> List[Dict]: p = dict(params) warmup_prepended = 0 rest_warmup_stats: Dict[str, int] = {} period_start_key = "" if meta_out is not None: start_key_meta = str(meta_out.get("start_key") or "") if len(start_key_meta) >= 12: period_start_key = start_key_meta[:12] p["_backtest_period_start_key"] = period_start_key db_meta = meta_out.get("db") if db_meta is not None and period_start_key: warmup_prepended = prepend_momentum_candle_warmup( db_meta, candles_by_code, period_start_key, ) if not period_start_key: period_start_key = str(p.get("_backtest_period_start_key") or "")[:12] if len(period_start_key) >= 8: # 해외 US 티커는 키움 분봉 REST 불가 — 유량 낭비·실패 폭주 방지 if str(p.get("market") or "").strip().upper() != "US": rest_warmup_stats = inject_momentum_rest_warmup_memory( candles_by_code, period_start_key, universe_by_slot=universe_by_slot, ) p["slot_money"] = float(slot_money) p["fee_rate"] = float(fee_rate) p["sell_tax"] = float(sell_tax) # 해외 환전: params 에 있으면 유지 (Optuna base_fixed / 웹 US params) if "fx_fee_rate" not in p: p["fx_fee_rate"] = 0.0 p["max_stocks"] = int(max_stocks) if total_budget_krw > 0: p["total_budget_krw"] = float(total_budget_krw) p.setdefault("portfolio_mode", True) loaded_ticks: Dict[str, Dict[str, List[Dict]]] = dict(ticks_by_code or {}) tick_meta: Dict[str, Any] = {} from kis_trader.engine.momentum_tick_replay import ( momentum_backtest_use_tick_entry, momentum_backtest_use_tick_exit, ) if momentum_backtest_use_tick_exit(p) or momentum_backtest_use_tick_entry(p): from kis_trader.backtest.momentum_tick_loader import ( load_momentum_ticks_by_code, tick_coverage_stats, ) if not loaded_ticks and meta_out is not None: start_key = str(meta_out.get("start_key") or "") end_key = str(meta_out.get("end_key") or "") db = meta_out.get("db") if db is None and start_key and end_key: from kis_trader.backtest.backtest_portfolio_common import ensure_meta_db db = ensure_meta_db(meta_out) if db and start_key and end_key: _mkt = str(p.get("market") or "KR").strip().upper() or "KR" loaded_ticks, tick_rows = load_momentum_ticks_by_code( db, start_key, end_key, set(candles_by_code.keys()), market=_mkt, ) tick_meta = tick_coverage_stats(candles_by_code, loaded_ticks) tick_meta["ws_tick_rows_loaded"] = tick_rows tick_meta["ws_ticks_table"] = "ws_ticks_us" if _mkt == "US" else "ws_ticks" if tick_rows <= 0: from kis_trader.utils.logger import get_logger as _get_logger _tick_tbl = tick_meta["ws_ticks_table"] _get_logger("kis_trader.momentum_backtest").warning( "⚠️ %s 데이터 없음 — 1분봉 OHLC 청산 폴백 (틱 수집 후 재백테 권장)", _tick_tbl, ) elif loaded_ticks: tick_meta = tick_coverage_stats(candles_by_code, loaded_ticks) tick_meta["ws_tick_rows_loaded"] = sum( len(lst) for cm in loaded_ticks.values() for lst in cm.values() ) # ── 틱 전무 종목 제외 (실매 정합) ────────────────────────────── # 틱이 하나도 없는 종목은 개장 순간 반짝 후보(유니버스 2분 존속)라 실제 # 매매 불가. 남겨두면 OHLC 폴백으로 유령 거래를 만들어 순위를 오염시킨다. from kis_trader.engine.momentum_tick_replay import ( momentum_backtest_tick_only_codes, ) if loaded_ticks and momentum_backtest_tick_only_codes(p): tick_codes = {c for c, m in loaded_ticks.items() if any(m.values())} before_n = len(candles_by_code) dropped = [c for c in candles_by_code if c not in tick_codes] if dropped and len(tick_codes) > 0: candles_by_code = { c: v for c, v in candles_by_code.items() if c in tick_codes } tick_meta["tick_only_codes_dropped"] = len(dropped) tick_meta["tick_only_codes_kept"] = len(candles_by_code) from kis_trader.utils.logger import get_logger as _get_logger _get_logger("kis_trader.momentum_backtest").info( "🎯 틱 전무 종목 %d개 제외 (%d→%d종목) — 틱 있는 종목만 백테", len(dropped), before_n, len(candles_by_code), ) ob_loaded, pg_loaded, snap_meta = resolve_trigger_snapshots_for_backtest( candles_by_code, p, strategy="MOMENTUM", meta_out=meta_out, orderbook_by_code=orderbook_by_code, program_by_code=program_by_code, ) if snap_meta.get("log_verdict_by_code"): p["_backtest_log_verdict_by_code"] = snap_meta["log_verdict_by_code"] if meta_out is not None: start_key = str(meta_out.get("start_key") or "") end_key = str(meta_out.get("end_key") or "") if len(start_key) >= 8 and len(end_key) >= 8: from kis_trader.backtest.momentum_universe_timeline import ( attach_momentum_universe_timeline_to_params, ) attach_momentum_universe_timeline_to_params( p, start_ymd=start_key[:8], end_ymd=end_key[:8], strategy_id=MOMENTUM_STRATEGY_ID, use_saved_history=universe_by_slot is not None, ) from kis_trader.backtest.backtest_env_timeline import attach_backtest_env_timeline_to_params attach_backtest_env_timeline_to_params(p, meta_out, MOMENTUM_STRATEGY_ID) trades = run_momentum_backtest( candles_by_code, p, universe_by_slot=universe_by_slot, ticks_by_code=loaded_ticks or ticks_by_code, orderbook_by_code=ob_loaded, program_by_code=pg_loaded, ) if not p.get("portfolio_mode"): _fx = float(p.get("fx_fee_rate", 0.0) or 0.0) _us = str(p.get("market") or "").strip().upper() == "US" attach_scalp_trade_pnl( trades, fee_rate=fee_rate, sell_tax=sell_tax, slip_pct=backtest_slip_pct(p), fx_fee_rate=_fx, pnl_decimals=4 if (_us or _fx > 0) else 0, ) if meta_out is not None: meta_out["skip_stats"] = p.get("_portfolio_skip_stats") or {} if warmup_prepended > 0 or momentum_backtest_candle_warmup_bars() > 0: meta_out["skip_stats"]["candle_warmup_bars"] = momentum_backtest_candle_warmup_bars() meta_out["skip_stats"]["candle_warmup_prepended"] = warmup_prepended if rest_warmup_stats: meta_out["skip_stats"]["rest_warmup"] = rest_warmup_stats meta_out["universe_timing"] = ( "strict" if momentum_backtest_universe_strict_enabled() else "minute" ) if p.get("_universe_timeline_meta"): meta_out["universe_timeline"] = p.get("_universe_timeline_meta") if snap_meta: meta_out["trigger_snapshot_backtest"] = snap_meta if tick_meta: from kis_trader.backtest.breakout_tick_loader import enrich_tick_meta_with_traded_codes tick_meta = enrich_tick_meta_with_traded_codes( tick_meta, candles_by_code, loaded_ticks, trades, ) meta_out["tick_backtest"] = tick_meta return trades def summarize_momentum_trades( trades: List[Dict], *, total_budget_krw: float, period_days: int, ) -> Dict[str, Any]: return summarize_trades(trades, total_budget_krw=total_budget_krw, period_days=period_days) def count_momentum_sell_reasons(trades: List[Dict]) -> Dict[str, int]: out: Dict[str, int] = {} for t in trades: r = str(t.get("sell_reason") or "기타") out[r] = out.get(r, 0) + 1 return out