kis_bot/kis_trader/backtest/momentum_backtest_common.py

#!/usr/bin/env python3
"""
모멘텀 백테스트 공통 — backtest_web / param_search 단일 진입점.

실매 MomentumStrategy 와 동일:
  - 진입: ``momentum_engine.check_buy_signal_momentum_live``
  - 청산: ws_ticks 틱 리플레이(실매 체결가) 또는 ``check_sell_signal_momentum_backtest_bar`` 폴백
"""
from __future__ import annotations

from typing import Any, Dict, List, Optional, Tuple

from kis_trader.backtest.backtest_portfolio_common import (
    attach_scalp_trade_pnl,
    backtest_slip_pct,
    build_budget_warning,
    fee_and_slot_from_env_row,
    merge_portfolio_into_params,
    resolve_portfolio_params,
    resolve_trigger_snapshots_for_backtest,
    summarize_trades,
)
from kis_trader.engine.momentum_engine import (
    MOMENTUM_STRATEGY_ID,
    get_momentum_defaults_from_db,
    run_momentum_backtest,
)
from kis_trader.engine.indicator_cache import (
    materialize_ws_candles_batch,
    ws_candles_select_indicator_cols,
)

MOMENTUM_STRATEGY = MOMENTUM_STRATEGY_ID


def momentum_backtest_universe_strict_enabled() -> bool:
    """모멘텀 백테: 종목별 첫 event_time 이후 분만 유니버스 (실매 정합)."""
    from kis_trader.utils.env import get_env_bool
    return get_env_bool("MOMENTUM_BACKTEST_UNIVERSE_STRICT", True)


def momentum_backtest_universe_strict_lag_min() -> int:
    """첫 편입 분 이후 추가 대기 분 (기본 1 → 09:42:25 편입은 09:43 분봉부터)."""
    from kis_trader.utils.env import get_env_int
    return max(0, int(get_env_int("MOMENTUM_BACKTEST_UNIVERSE_STRICT_LAG_MIN", 1)))


def momentum_universe_exit_debounce_sec() -> int:
    """
    백테 EXIT 디바운스(초).

    실매 ``CONDITION_EXIT_GRACE_SEC``(기본 120)가 이미 스냅샷에 반영되어 있으면
    이중 디바운스를 쓰지 않음 — env ``MOMENTUM_UNIVERSE_EXIT_DEBOUNCE_SEC`` 로만 강제 가능.
    """
    from kis_trader.utils.env import get_env_from_db, get_env_int
    raw = get_env_from_db("MOMENTUM_UNIVERSE_EXIT_DEBOUNCE_SEC", None)
    if raw not in (None, ""):
        try:
            return max(0, int(float(raw)))
        except (TypeError, ValueError):
            pass
    grace = int(get_env_int("CONDITION_EXIT_GRACE_SEC", 0))
    if grace > 0:
        return 0
    return max(0, int(get_env_int("MOMENTUM_UNIVERSE_EXIT_DEBOUNCE_SEC", 30)))


def momentum_backtest_candle_warmup_bars() -> int:
    """백테 지표 warm-up — 실매 ``get_candles(50)`` 과 동일하게 전일·당일 장전 봉 선행."""
    from kis_trader.utils.env import get_env_int
    return max(0, int(get_env_int("MOMENTUM_BACKTEST_CANDLE_WARMUP_BARS", 50)))


def prepend_momentum_candle_warmup(
    db,
    candles_by_code: Dict[str, List[Dict]],
    period_start_key: str,
    *,
    warmup_bars: Optional[int] = None,
) -> int:
    """
    ``period_start_key``(YYYYMMDDHHMM) 이전 N봉을 종목별로 prepend.
    RSI·EMA·패턴 판별용 — 루프 시각(all_times)에는 포함하지 않음.
    """
    wb = (
        momentum_backtest_candle_warmup_bars()
        if warmup_bars is None
        else max(0, int(warmup_bars))
    )
    if wb <= 0 or db is None or not period_start_key:
        return 0
    ps = str(period_start_key)[:12]
    ind_cols = ws_candles_select_indicator_cols(db)
    total_prepended = 0
    for code, rows in list(candles_by_code.items()):
        if not rows:
            continue
        first_period_idx = None
        for i, r in enumerate(rows):
            ct = str(r.get("candle_time") or "")
            if ct >= ps:
                first_period_idx = i
                break
        if first_period_idx is None:
            continue
        if first_period_idx > 0:
            continue
        first_ct = str(rows[first_period_idx].get("candle_time") or "")
        if not first_ct:
            continue
        warm_rows = db.conn.execute(
            f"SELECT candle_time, open, high, low, close, volume, is_confirmed{ind_cols} "
            "FROM ws_candles WHERE timeframe=1 AND code=%s "
            "AND candle_time < %s ORDER BY candle_time DESC LIMIT %s",
            [code, first_ct, wb],
        ).fetchall()
        if not warm_rows:
            continue
        prefix = [dict(r) for r in reversed(warm_rows)]
        candles_by_code[code] = prefix + [dict(r) for r in rows]
        total_prepended += len(prefix)
    if total_prepended > 0:
        materialize_ws_candles_batch(db, candles_by_code, 1)
    return total_prepended


def date_keys(start: str, end: str) -> Tuple[str, str, str, str]:
    start_key = start.replace("-", "") + "0000"
    end_key = end.replace("-", "") + "2359"
    return start_key, end_key, start_key[:8], end_key[:8]


def resolve_momentum_universe(
    start_ymd: str,
    end_ymd: str,
    *,
    use_saved_history: bool,
    strategy_id: str = MOMENTUM_STRATEGY_ID,
) -> Tuple[Optional[Dict[str, List[str]]], str, int, int, str]:
    """
    Returns:
        universe_by_slot, source_label, slot_count, scan_interval_min, universe_timing
        universe_timing: 'strict' | 'minute' | 'all'
    """
    if use_saved_history and strategy_id:
        try:
            from kis_trader.database.db_manager import get_db as _get_ext_db
            strict = momentum_backtest_universe_strict_enabled()
            lag_min = momentum_backtest_universe_strict_lag_min()
            debounce_sec = momentum_universe_exit_debounce_sec()
            history = _get_ext_db().get_universe_by_candle_time(
                strategy_id=strategy_id,
                start_ymd=start_ymd,
                end_ymd=end_ymd,
                strict=strict,
                strict_lag_minutes=lag_min,
                exit_debounce_sec=debounce_sec,
            )
            if history:
                timing = "strict" if strict else "minute"
                label = "history_strict" if strict else "history"
                return history, label, len(history), 1, timing
        except Exception:
            pass
    return None, "all", 0, 1, "all"


def load_momentum_candles_by_code(
    db,
    start_key: str,
    end_key: str,
    *,
    warmup_bars: Optional[int] = None,
) -> Tuple[Dict[str, List[Dict]], int]:
    period_start = str(start_key)[:12]
    codes_raw = db.conn.execute(
        "SELECT DISTINCT code FROM ws_candles WHERE timeframe=1 "
        "AND candle_time >= %s AND candle_time <= %s ORDER BY code",
        [start_key, end_key],
    ).fetchall()
    codes = [r["code"] for r in codes_raw]
    ind_cols = ws_candles_select_indicator_cols(db)
    candles_by_code: Dict[str, List[Dict]] = {}
    total = 0
    for code in codes:
        rows = db.conn.execute(
            f"SELECT candle_time, open, high, low, close, volume, is_confirmed{ind_cols} "
            "FROM ws_candles WHERE timeframe=1 AND code=%s "
            "AND candle_time >= %s AND candle_time <= %s "
            "ORDER BY candle_time ASC",
            [code, start_key, end_key],
        ).fetchall()
        if len(rows) < 6:
            continue
        candles_by_code[code] = [dict(r) for r in rows]
        total += len(rows)
    prepend_momentum_candle_warmup(
        db, candles_by_code, period_start, warmup_bars=warmup_bars,
    )
    materialize_ws_candles_batch(db, candles_by_code, 1)
    return candles_by_code, total


def resolve_momentum_portfolio_params(
    env_row: Dict[str, Any],
    base_params: Dict[str, Any],
    *,
    slot_money: Optional[float] = None,
    max_stocks: Optional[int] = None,
    total_budget_krw: Optional[float] = None,
) -> Dict[str, Any]:
    return resolve_portfolio_params(
        env_row,
        base_params,
        strategy="MOMENTUM",
        slot_money=slot_money,
        max_stocks=max_stocks,
        total_budget_krw=total_budget_krw,
    )


def run_momentum_backtest_web_aligned(
    candles_by_code: Dict[str, List[Dict]],
    params: Dict[str, Any],
    universe_by_slot: Optional[Dict[str, List[str]]] = None,
    *,
    slot_money: float = 3_000_000.0,
    fee_rate: float = 0.00015,
    sell_tax: float = 0.0018,
    max_stocks: int = 3,
    total_budget_krw: float = 0.0,
    ticks_by_code: Optional[Dict[str, Dict[str, List[Dict]]]] = None,
    orderbook_by_code: Optional[Dict[str, Dict[str, List[Any]]]] = None,
    program_by_code: Optional[Dict[str, Dict[str, List[Any]]]] = None,
    meta_out: Optional[Dict[str, Any]] = None,
) -> List[Dict]:
    p = dict(params)
    warmup_prepended = 0
    period_start_key = ""
    if meta_out is not None:
        start_key_meta = str(meta_out.get("start_key") or "")
        if len(start_key_meta) >= 12:
            period_start_key = start_key_meta[:12]
            p["_backtest_period_start_key"] = period_start_key
        db_meta = meta_out.get("db")
        if db_meta is not None and period_start_key:
            warmup_prepended = prepend_momentum_candle_warmup(
                db_meta, candles_by_code, period_start_key,
            )
    p["slot_money"] = float(slot_money)
    p["fee_rate"] = float(fee_rate)
    p["sell_tax"] = float(sell_tax)
    p["max_stocks"] = int(max_stocks)
    if total_budget_krw > 0:
        p["total_budget_krw"] = float(total_budget_krw)
    p.setdefault("portfolio_mode", True)

    loaded_ticks: Dict[str, Dict[str, List[Dict]]] = dict(ticks_by_code or {})
    tick_meta: Dict[str, Any] = {}
    from kis_trader.engine.momentum_tick_replay import (
        momentum_backtest_use_tick_entry,
        momentum_backtest_use_tick_exit,
    )
    if momentum_backtest_use_tick_exit(p) or momentum_backtest_use_tick_entry(p):
        from kis_trader.backtest.momentum_tick_loader import (
            load_momentum_ticks_by_code,
            tick_coverage_stats,
        )
        if not loaded_ticks and meta_out is not None:
            start_key = str(meta_out.get("start_key") or "")
            end_key = str(meta_out.get("end_key") or "")
            db = meta_out.get("db")
            if db and start_key and end_key:
                loaded_ticks, tick_rows = load_momentum_ticks_by_code(
                    db, start_key, end_key, set(candles_by_code.keys()),
                )
                tick_meta = tick_coverage_stats(candles_by_code, loaded_ticks)
                tick_meta["ws_tick_rows_loaded"] = tick_rows
                if tick_rows <= 0:
                    from kis_trader.utils.logger import get_logger as _get_logger

                    _get_logger("kis_trader.momentum_backtest").warning(
                        "⚠️ ws_ticks 데이터 없음 — 1분봉 OHLC 청산 폴백 (틱 수집 후 재백테 권장)",
                    )
        elif loaded_ticks:
            tick_meta = tick_coverage_stats(candles_by_code, loaded_ticks)
            tick_meta["ws_tick_rows_loaded"] = sum(
                len(lst) for cm in loaded_ticks.values() for lst in cm.values()
            )

        # ── 틱 전무 종목 제외 (실매 정합) ──────────────────────────────
        # 틱이 하나도 없는 종목은 개장 순간 반짝 후보(유니버스 2분 존속)라 실제
        # 매매 불가. 남겨두면 OHLC 폴백으로 유령 거래를 만들어 순위를 오염시킨다.
        from kis_trader.engine.momentum_tick_replay import (
            momentum_backtest_tick_only_codes,
        )
        if loaded_ticks and momentum_backtest_tick_only_codes(p):
            tick_codes = {c for c, m in loaded_ticks.items() if any(m.values())}
            before_n = len(candles_by_code)
            dropped = [c for c in candles_by_code if c not in tick_codes]
            if dropped and len(tick_codes) > 0:
                candles_by_code = {
                    c: v for c, v in candles_by_code.items() if c in tick_codes
                }
                tick_meta["tick_only_codes_dropped"] = len(dropped)
                tick_meta["tick_only_codes_kept"] = len(candles_by_code)
                from kis_trader.utils.logger import get_logger as _get_logger

                _get_logger("kis_trader.momentum_backtest").info(
                    "🎯 틱 전무 종목 %d개 제외 (%d→%d종목) — 틱 있는 종목만 백테",
                    len(dropped), before_n, len(candles_by_code),
                )

    ob_loaded, pg_loaded, snap_meta = resolve_trigger_snapshots_for_backtest(
        candles_by_code, p, strategy="MOMENTUM", meta_out=meta_out,
        orderbook_by_code=orderbook_by_code, program_by_code=program_by_code,
    )
    if snap_meta.get("log_verdict_by_code"):
        p["_backtest_log_verdict_by_code"] = snap_meta["log_verdict_by_code"]
    if meta_out is not None:
        start_key = str(meta_out.get("start_key") or "")
        end_key = str(meta_out.get("end_key") or "")
        if len(start_key) >= 8 and len(end_key) >= 8:
            from kis_trader.backtest.momentum_universe_timeline import (
                attach_momentum_universe_timeline_to_params,
            )
            attach_momentum_universe_timeline_to_params(
                p,
                start_ymd=start_key[:8],
                end_ymd=end_key[:8],
                strategy_id=MOMENTUM_STRATEGY_ID,
                use_saved_history=universe_by_slot is not None,
            )
    trades = run_momentum_backtest(
        candles_by_code, p, universe_by_slot=universe_by_slot,
        ticks_by_code=loaded_ticks or ticks_by_code,
        orderbook_by_code=ob_loaded,
        program_by_code=pg_loaded,
    )
    if not p.get("portfolio_mode"):
        attach_scalp_trade_pnl(
            trades, fee_rate=fee_rate, sell_tax=sell_tax,
            slip_pct=backtest_slip_pct(p),
        )
    if meta_out is not None:
        meta_out["skip_stats"] = p.get("_portfolio_skip_stats") or {}
        if warmup_prepended > 0 or momentum_backtest_candle_warmup_bars() > 0:
            meta_out["skip_stats"]["candle_warmup_bars"] = momentum_backtest_candle_warmup_bars()
            meta_out["skip_stats"]["candle_warmup_prepended"] = warmup_prepended
        meta_out["universe_timing"] = (
            "strict" if momentum_backtest_universe_strict_enabled() else "minute"
        )
        if p.get("_universe_timeline_meta"):
            meta_out["universe_timeline"] = p.get("_universe_timeline_meta")
        if snap_meta:
            meta_out["trigger_snapshot_backtest"] = snap_meta
        if tick_meta:
            meta_out["tick_backtest"] = tick_meta
    return trades


def summarize_momentum_trades(
    trades: List[Dict],
    *,
    total_budget_krw: float,
    period_days: int,
) -> Dict[str, Any]:
    return summarize_trades(trades, total_budget_krw=total_budget_krw, period_days=period_days)


def count_momentum_sell_reasons(trades: List[Dict]) -> Dict[str, int]:
    out: Dict[str, int] = {}
    for t in trades:
        r = str(t.get("sell_reason") or "기타")
        out[r] = out.get(r, 0) + 1
    return out