ls증권 히스토리 구독 넣음

kis_trader/backtest/optuna_momentum.py

@@ -17,8 +17,19 @@ from database import TradeDB
 from kis_trader.backtest import momentum_backtest_common as mbc
 from kis_trader.backtest import scalping_backtest_common as sbc
 from kis_trader.backtest.optuna_search_space import momentum_grid_axis_keys, suggest_momentum_params
+from kis_trader.backtest.optuna_momentum_tpe_space import (
+    momentum_tpe_axis_keys,
+    suggest_momentum_params_tpe,
+)
 from kis_trader.backtest.optuna_mode_combo import enrich_out_data_with_mode_combo
-from kis_trader.backtest.optuna_common import announce_optuna_json_path, release_shared_tick_store
+from kis_trader.backtest.optuna_common import (
+    announce_optuna_json_path,
+    build_optuna_result_tiers,
+    pick_gated_apply_trial,
+    release_shared_tick_store,
+    set_optuna_trial_stability_attrs,
+    stability_fields_from_trial_attrs,
+)
 from kis_trader.backtest.param_search_cli_common import (
     apply_session_to_fixed,
     combo_passes_search_filters,
@@ -66,6 +77,9 @@ class MomentumSearchContext:
     end_key: str
     cache_holder: Dict[str, Any] = field(default_factory=dict)
     shared_tick_store: Any = None  # ws_ticks 공유메모리 핸들 (종료 시 unlink)
+    market: str = "KR"
+    # 종목 cfg Optuna: 1종목 유니버스 (없으면 전역)
+    symbol: str = ""
 
 
 def prepare_momentum_search_context(
@@ -80,13 +94,39 @@ def prepare_momentum_search_context(
     max_stocks: Optional[int] = None,
     total_budget_krw: Optional[float] = None,
     orderbook_filter: str = "off",
+    market: str = "KR",
+    codes_filter: Optional[List[str]] = None,
+    symbol: Optional[str] = None,
+    history_source: Optional[str] = None,
 ) -> Optional[MomentumSearchContext]:
-    grids = _momentum_grids()
-    if mode not in grids:
-        logger.error("❌ 모멘텀 mode: %s (fast/exit/rr/coarse/fine/wide/full)", mode)
+    mk = (market or "KR").strip().upper() or "KR"
+    sym = str(symbol or "").strip().upper()
+    filt: Optional[List[str]] = None
+    if codes_filter:
+        filt = [str(c).strip().upper() for c in codes_filter if str(c).strip()]
+    elif sym:
+        filt = [sym]
+    grids = _momentum_grids(market=mk if mk in ("US", "KR") else "KR")
+    # tpe = 연속 Optuna 전용 (Grid 메뉴 미사용). 기존 fast/fine/… 는 그대로.
+    if mode == "tpe":
+        grid: Dict[str, Any] = {}
+    elif mode not in grids:
+        logger.error(
+            "❌ 모멘텀 mode: %s (fast/exit/rr/coarse/fine/wide/full/tpe)",
+            mode,
+        )
         return None
+    else:
+        grid = grids[mode]
 
-    base_fixed = _mom_fixed_defaults()
+    base_fixed = _mom_fixed_defaults(market=mk if mk in ("US", "KR") else "KR")
+    if mode == "tpe":
+        # 연속 탐색도 HTS skip 스윕 금지 (KR=false 고정 / US=True 고정 — HTS 없음)
+        if mk != "US":
+            base_fixed["skip_hts_scan_dupes"] = False
+        logger.info(
+            "📌 mode=tpe — 연속(float/int) 탐색 (Grid categorical 미사용, TPE 가 구간 축소)"
+        )
     apply_session_to_fixed(base_fixed, time_start_hm=time_start_hm, time_end_hm=time_end_hm)
 
     from kis_trader.engine.momentum_tick_replay import (
@@ -95,17 +135,34 @@ def prepare_momentum_search_context(
     )
     base_fixed["backtest_use_tick_entry"] = _te(None)
     base_fixed["backtest_use_tick_exit"] = _tx(None)
+    # 절대규칙: Optuna OHLC 폴백으로 숫자 변조 금지
+    base_fixed["backtest_tick_fallback_ohlc"] = False
 
     _ob_mode = (orderbook_filter or "off").strip().lower()
+    if mk == "US":
+        # 해외: 호가 없음 · HTS 없음 · 자정 넘김 · 매매세 0 기본
+        _ob_mode = "off"
+        use_fallback_universe = True
+        base_fixed["_session_wrap_midnight"] = True
+        # skip_hts 는 _overlay_us 에서 True (HTS 유니버스 없음)
+        base_fixed["market"] = "US"
+        # fee/tax 는 _overlay_us 가 UI% 로 이미 세팅. 여기서 비율로 덮어쓰지 않음.
+        # 환전 편도 비율은 엔진 파라미터로 유지 (_ui_to_engine 이 나누지 않음).
+        try:
+            from kis_trader.engine.us_momentum_env_keys import us_momentum_trading_cost_rates
+            base_fixed["fx_fee_rate"] = float(us_momentum_trading_cost_rates()["fx_fee_rate"])
+        except Exception:
+            base_fixed["fx_fee_rate"] = float(get_env_float("US_MOMENTUM_FX_FEE_RATE", 0.0005))
     if _ob_mode == "off":
         base_fixed["_orderbook_filter_enabled"] = False
     elif _ob_mode == "on":
         base_fixed["_orderbook_filter_enabled"] = True
     ob_filter_on = bool(base_fixed.get("_orderbook_filter_enabled")) or _ob_mode == "auto"
     logger.info(
-        "📌 호가필터: %s (%s)",
+        "📌 호가필터: %s (%s) market=%s",
         _ob_mode.upper(),
         "적용" if ob_filter_on else "스킵 — 코어 파라미터 순수 탐색",
+        mk,
     )
 
     db = TradeDB()
@@ -115,11 +172,25 @@ def prepare_momentum_search_context(
     finally:
         db.close()
 
-    fee_rate, sell_tax, slot_from_env = sbc.fee_and_slot_from_env(env_row, strategy="MOMENTUM")
+    fee_rate, sell_tax, slot_from_env = sbc.fee_and_slot_from_env(
+        env_row, strategy="US_MOMENTUM" if mk == "US" else "MOMENTUM",
+    )
+    if mk == "US":
+        # overlay UI% → 비율 (attach_scalp_trade_pnl / portfolio)
+        fee_rate = float(base_fixed.get("fee_rate", 0.25)) / 100.0
+        sell_tax = float(base_fixed.get("sell_tax", 0.00206)) / 100.0
+        if fee_rate > 1.0:
+            fee_rate = fee_rate / 100.0
+        if sell_tax > 1.0:
+            sell_tax = sell_tax / 100.0
+    # 종목 cfg Optuna: 유니버스 1종 → 동시보유 1 고정 (전역 포트와 혼동 방지)
+    _ms_arg = max_stocks
+    if filt and len(filt) == 1 and _ms_arg is None:
+        _ms_arg = 1
     portfolio = sbc.resolve_scalp_portfolio_params(
-        env_row, None, strategy="MOMENTUM",
+        env_row, None, strategy="US_MOMENTUM" if mk == "US" else "MOMENTUM",
         slot_money=slot_money if slot_money is not None else slot_from_env,
-        max_stocks=max_stocks,
+        max_stocks=_ms_arg,
         total_budget_krw=total_budget_krw,
     )
     slot_money_f = float(portfolio["slot_money"])
@@ -131,14 +202,24 @@ def prepare_momentum_search_context(
     )
     logger.info(
         f"💼 포트폴리오: 1회 {slot_money_f:,.0f}원 | 동시 {max_stocks_i}종 | "
-        f"총한도 {total_budget_f:,.0f}원 | 매매 {format_session_hm(base_fixed)}"
+        f"총한도 {total_budget_f:,.0f}원 | 매매 {format_session_hm(base_fixed)} | market={mk}"
     )
 
-    codes_candles = _load_candles_for_search(start, end, base_fixed.get("rsi_period", 3))
+    codes_candles = _load_candles_for_search(
+        start, end, base_fixed.get("rsi_period", 3),
+        market=mk if mk in ("US", "KR") else None,
+        codes_filter=filt,
+    )
     if not codes_candles:
-        logger.error("❌ 캔들 데이터 없음")
+        logger.error("❌ 캔들 데이터 없음 (market=%s filt=%s)", mk, filt)
         return None
-    logger.info("✅ 데이터 로드: %s종목", len(codes_candles))
+    if filt and len(codes_candles) == 1:
+        logger.info(
+            "✅ 종목 Optuna 유니버스: %s (1종 · market=%s)",
+            next(iter(codes_candles.keys())), mk,
+        )
+    else:
+        logger.info("✅ 데이터 로드: %s종목 (market=%s)", len(codes_candles), mk)
 
     start_key = (start.replace("-", "") + "0000") if start else "202601010000"
     end_key = (end.replace("-", "") + "2359") if end else "999912312359"
@@ -147,22 +228,37 @@ def prepare_momentum_search_context(
 
     universe_by_slot = None
     fallback_sim_interval = 5
-    if not use_fallback_universe and start_ymd and end_ymd:
+    if mk == "US":
+        # 영구구독 US — HTS history 없음 · 시뮬 유니버스 스킵(봉에 있는 US 종목 전부)
+        use_fallback_universe = True
+        universe_by_slot = None
+        base_fixed["scan_interval_min"] = 1
+        logger.info("📌 US 모멘텀 Optuna — 영구구독/봉 유니버스 (HTS·시뮬 미사용)")
+    elif not use_fallback_universe and start_ymd and end_ymd:
         try:
             from kis_trader.backtest.momentum_backtest_common import resolve_momentum_universe
+            from kis_trader.backtest.universe_history_source import (
+                resolve_backtest_universe_history_source,
+            )
+
+            # scan_at 타임라인과 슬롯 dict 가 같은 이력소스(키움/LS)를 쓰도록 스태시
+            _hs = resolve_backtest_universe_history_source(history_source)
+            base_fixed["_universe_history_source"] = _hs
             history, src, n_bins, _scan_iv, timing = resolve_momentum_universe(
                 start_ymd, end_ymd, use_saved_history=True, strategy_id="MOMENTUM",
+                history_source=_hs,
             )
             if history:
                 universe_by_slot = history
                 avg = sum(len(v) for v in history.values()) / max(1, n_bins)
                 logger.info(
-                    "✅ 유니버스: MOMENTUM 이력 | %s분봉 · 평균 %.1f종목", n_bins, avg,
+                    "✅ 유니버스: MOMENTUM 이력 src=%s | %s분봉 · 평균 %.1f종목",
+                    src, n_bins, avg,
                 )
         except Exception as exc:
             logger.debug("유니버스 이력 스킵: %s", exc)
 
-    if universe_by_slot is None:
+    if universe_by_slot is None and mk != "US":
         universe_top_n = int(os.environ.get("UPDATE_UNIVERSE_TOP_N", "20"))
         universe_min_score = float(os.environ.get("UPDATE_UNIVERSE_MIN_SCORE", "4.0"))
         universe_by_slot = me.build_universe_simulation_momentum(
@@ -173,28 +269,30 @@ def prepare_momentum_search_context(
         )
         base_fixed["scan_interval_min"] = fallback_sim_interval
         logger.info("📌 유니버스: 모멘텀 시뮬 fallback (%d분)", fallback_sim_interval)
-    else:
+    elif universe_by_slot is not None:
         base_fixed["scan_interval_min"] = 1
 
     # DB 전일봉 없으면 키움 REST 1회/종목 → 메모리 prepend (실매 갭보정 정합, DB 미기록)
-    try:
-        from kis_trader.backtest.momentum_backtest_common import (
-            inject_momentum_rest_warmup_memory,
-        )
-        _rw = inject_momentum_rest_warmup_memory(
-            codes_candles,
-            start_key,
-            universe_by_slot=universe_by_slot,
-        )
-        if int(_rw.get("ok") or 0) > 0 or int(_rw.get("need") or 0) > 0:
-            logger.info(
-                "📡 REST 웜업: need=%s ok=%s fail=%s bars=%s",
-                _rw.get("need"), _rw.get("ok"), _rw.get("fail"), _rw.get("bars"),
+    # 해외 US: 키움 분봉 불가 → REST 웜업 스킵
+    if mk != "US":
+        try:
+            from kis_trader.backtest.momentum_backtest_common import (
+                inject_momentum_rest_warmup_memory,
             )
-    except Exception as exc:
-        logger.warning("⚠️ REST 웜업 스킵: %s", exc)
+            _rw = inject_momentum_rest_warmup_memory(
+                codes_candles,
+                start_key,
+                universe_by_slot=universe_by_slot,
+            )
+            if int(_rw.get("ok") or 0) > 0 or int(_rw.get("need") or 0) > 0:
+                logger.info(
+                    "📡 REST 웜업: need=%s ok=%s fail=%s bars=%s",
+                    _rw.get("need"), _rw.get("ok"), _rw.get("fail"), _rw.get("bars"),
+                )
+        except Exception as exc:
+            logger.warning("⚠️ REST 웜업 스킵: %s", exc)
 
-    grid = grids[mode]
+    # grid 는 상단에서 mode별 설정 (tpe=빈 dict). 여기서 grids[mode] 재조회하면 tpe KeyError.
     _ob_axes = ("max_spread_pct", "min_bid_ask_ratio", "ask_max_mult")
     _ob_sweeping = any(len(set(grid.get(k) or [])) > 1 for k in _ob_axes)
     if ob_filter_on and _ob_sweeping:
@@ -225,8 +323,10 @@ def prepare_momentum_search_context(
             from kis_trader.backtest.momentum_tick_loader import load_momentum_ticks_by_code
             ticks_by_code, tick_rows = load_momentum_ticks_by_code(
                 _snap_db, start_key, end_key, set(codes_candles.keys()),
+                market=mk,
             )
-            logger.info("✅ ws_ticks %s건", f"{tick_rows:,}")
+            _tick_tbl = "ws_ticks_us" if mk == "US" else "ws_ticks"
+            logger.info("✅ %s %s건 (market=%s)", _tick_tbl, f"{tick_rows:,}", mk)
     finally:
         _snap_db.close()
 
@@ -270,11 +370,15 @@ def prepare_momentum_search_context(
         total_budget_krw=total_budget_f,
         period_days=period_days,
         portfolio=portfolio,
-        grid_keys=momentum_grid_axis_keys(mode),
+        grid_keys=(
+            momentum_tpe_axis_keys() if mode == "tpe" else momentum_grid_axis_keys(mode, market=mk)
+        ),
         start_key=start_key,
         end_key=end_key,
         cache_holder=cache_holder,
         shared_tick_store=shared_tick_store,
+        market=mk,
+        symbol=(sym or (filt[0] if filt and len(filt) == 1 else "")),
     )
 
 
@@ -282,7 +386,8 @@ def _make_sampler(name: str, seed: Optional[int]):
     n = (name or "tpe").strip().lower()
     if n == "random":
         return RandomSampler(seed=seed)
-    return TPESampler(seed=seed, multivariate=True)
+    # multivariate TPE + 조건부 suggest 시 independent sampling 경고가 trial마다 폭주 → 억제
+    return TPESampler(seed=seed, multivariate=True, warn_independent_sampling=False)
 
 
 def _momentum_objective_value(result: Dict[str, Any], sort_by: str) -> float:
@@ -320,7 +425,12 @@ def run_momentum_optuna(
     )
 
     def objective(trial: optuna.Trial) -> float:
-        combo = suggest_momentum_params(trial, ctx.mode)
+        if ctx.mode == "tpe":
+            combo = suggest_momentum_params_tpe(
+                trial, market=getattr(ctx, "market", "KR") or "KR",
+            )
+        else:
+            combo = suggest_momentum_params(trial, ctx.mode, market=getattr(ctx, "market", "KR") or "KR")
         result = evaluate_momentum_param_combo(
             combo,
             base_fixed=ctx.base_fixed,
@@ -356,6 +466,7 @@ def run_momentum_optuna(
         trial.set_user_attr("score", float(obj if sort_by == "score" else _momentum_objective_value(result, "score")))
         trial.set_user_attr("total_trades", int(result["total_trades"]))
         trial.set_user_attr("merged_json", json.dumps(result.get("merged_params") or {}, ensure_ascii=False))
+        set_optuna_trial_stability_attrs(trial, result)
         return float(obj)
 
     logger.info(
@@ -389,19 +500,10 @@ def run_momentum_optuna(
                 "score": float(trial.user_attrs.get("score") or 0),
                 "optuna_trial_number": trial.number,
             }
+            row.update(stability_fields_from_trial_attrs(trial))
             passing.append(row)
 
-        if sort_by == "score":
-            passing.sort(key=lambda r: (-r["score"], -r["total_pnl"], -r["win_rate"]))
-        elif sort_by == "win_rate":
-            passing.sort(key=lambda r: (-r["win_rate"], -r["total_pnl"]))
-        else:
-            passing.sort(key=lambda r: (-r["total_pnl"], -r["win_rate"]))
-
-        profitable = [r for r in passing if r["total_pnl"] > 0]
-        if profitable:
-            passing = profitable
-
+        tiers = build_optuna_result_tiers(passing, sort_by=sort_by)
         out_data = {
             "engine": "optuna",
             "strategy": "momentum",
@@ -425,16 +527,33 @@ def run_momentum_optuna(
             "optuna_best_value": study.best_value if study.best_trial else None,
             "optuna_best_trial_number": study.best_trial.number if study.best_trial else None,
             "elapsed_sec": round(elapsed, 1),
-            "results": passing[:5000],
+            **tiers,
         }
 
         ts = datetime.now().strftime("%Y%m%d_%H%M%S")
-        out_path = os.path.join(_results_dir_for_write(), f"optuna_momentum_{ctx.mode}_{ts}.json")
+        _mk = str(getattr(ctx, "market", "") or "").strip().upper()
+        _sym = str(getattr(ctx, "symbol", "") or "").strip().upper()
+        if _mk == "US" and _sym:
+            _fname = f"optuna_us_momentum_{_sym}_{ctx.mode}_{ts}.json"
+        elif _mk == "US":
+            _fname = f"optuna_us_momentum_{ctx.mode}_{ts}.json"
+        else:
+            _fname = f"optuna_momentum_{ctx.mode}_{ts}.json"
+        out_path = os.path.join(_results_dir_for_write(), _fname)
+        out_data["strategy"] = "us_momentum" if _mk == "US" else "momentum"
+        out_data["market"] = _mk or "KR"
+        if _sym:
+            out_data["symbol"] = _sym
+            out_data["_apply_symbol"] = _sym
         # 최빈 실측 전에 먼저 저장·경로 고지 (실측이 길어도 바로 파일 열 수 있게)
         with open(out_path, "w", encoding="utf-8") as f:
             json.dump(out_data, f, indent=2, ensure_ascii=False)
         announce_optuna_json_path(
-            out_path, strategy="momentum", mode=ctx.mode, note="중간저장(mode 전)", log=logger,
+            out_path,
+            strategy=("us_momentum" if _mk == "US" else "momentum"),
+            mode=ctx.mode,
+            note="중간저장(mode 전)",
+            log=logger,
         )
 
         def _eval_mode(combo: Dict[str, Any]) -> Optional[Dict[str, Any]]:
@@ -464,11 +583,17 @@ def run_momentum_optuna(
                 end_key=ctx.end_key,
             )
 
+        _ann_strat = "us_momentum" if _mk == "US" else "momentum"
+
         def _save_partial(_data: Dict[str, Any]) -> None:
             with open(out_path, "w", encoding="utf-8") as f:
                 json.dump(_data, f, indent=2, ensure_ascii=False)
             announce_optuna_json_path(
-                out_path, strategy="momentum", mode=ctx.mode, note="mode_combo params 저장(실측 전)", log=logger,
+                out_path,
+                strategy=_ann_strat,
+                mode=ctx.mode,
+                note="mode_combo params 저장(실측 전)",
+                log=logger,
             )
 
         enrich_out_data_with_mode_combo(
@@ -481,7 +606,7 @@ def run_momentum_optuna(
         with open(out_path, "w", encoding="utf-8") as f:
             json.dump(out_data, f, indent=2, ensure_ascii=False)
         announce_optuna_json_path(
-            out_path, strategy="momentum", mode=ctx.mode, note="최종 JSON", log=logger,
+            out_path, strategy=_ann_strat, mode=ctx.mode, note="최종 JSON", log=logger,
         )
         study._kis_export_path = out_path  # type: ignore[attr-defined]
         return study
@@ -491,15 +616,74 @@ def run_momentum_optuna(
 
 
 def apply_best_momentum_trial(study: optuna.Study) -> bool:
-    if not study.best_trial or study.best_value <= _FAIL_OBJECTIVE + 1:
-        logger.warning("⚠️ 적용할 best trial 없음")
+    # 탐색 best(objective)가 아니라 report_gates 통과 후보만 적용
+    trial = pick_gated_apply_trial(study, sort_by="score", fail_objective=_FAIL_OBJECTIVE)
+    if trial is None:
+        logger.warning(
+            "⚠️ 사후게이트(results_gated) 통과 trial 없음 — DB 미적용 "
+            "(탐색 min_wr/pf=0 이어도 apply 는 승률·PF 하한 필요)"
+        )
         return False
-    pnl = float(study.best_trial.user_attrs.get("total_pnl") or 0)
+    pnl = float(trial.user_attrs.get("total_pnl") or 0)
     if pnl <= 0:
-        logger.warning("⚠️ Best trial 총손익 ≤ 0 — DB 미적용")
+        logger.warning("⚠️ gated trial 총손익 ≤ 0 — DB 미적용")
         return False
-    merged_raw = study.best_trial.user_attrs.get("merged_json") or "{}"
+    merged_raw = trial.user_attrs.get("merged_json") or "{}"
     merged = json.loads(merged_raw)
     apply_params_to_db(merged)
-    logger.info("🚀 [Optuna apply-best] momentum trial #%d → env_config", study.best_trial.number)
+    logger.info("🚀 [Optuna apply-best] momentum gated trial #%d → env_config", trial.number)
+    try:
+        from kis_trader.backtest.optuna_daily_trail_recommend import (
+            apply_daily_trail_recommend_from_optuna_json,
+        )
+        apply_daily_trail_recommend_from_optuna_json(
+            getattr(study, "_kis_export_path", None),
+            strategy="momentum",
+            log=logger,
+        )
+    except Exception as exc:
+        logger.warning("⚠️ 다단트레일 추천 반영 스킵: %s", exc)
+    return True
+
+
+def apply_best_us_momentum_trial(study: optuna.Study, *, symbol: str = "") -> bool:
+    """해외 모멘텀 — 전역 config_us_momentum 또는 종목 stock_config.
+
+    symbol 있으면 ``us_momentum_stock_config`` 행만 갱신(전역·다단트레일 미오염).
+    """
+    from kis_trader.backtest.param_search_momentum import apply_params_to_db_us
+
+    sym = str(symbol or "").strip().upper()
+    trial = pick_gated_apply_trial(study, sort_by="score", fail_objective=_FAIL_OBJECTIVE)
+    if trial is None:
+        logger.warning("⚠️ us_momentum gated trial 없음 — DB 미적용")
+        return False
+    pnl = float(trial.user_attrs.get("total_pnl") or 0)
+    if pnl <= 0:
+        logger.warning("⚠️ us_momentum gated trial 총손익 ≤ 0 — DB 미적용")
+        return False
+    merged_raw = trial.user_attrs.get("merged_json") or "{}"
+    merged = json.loads(merged_raw)
+    apply_params_to_db_us(merged, symbol=sym)
+    if sym:
+        logger.info(
+            "🚀 [Optuna apply-best] us_momentum gated #%d → stock_config %s",
+            trial.number, sym,
+        )
+        return True
+    try:
+        from kis_trader.backtest.optuna_daily_trail_recommend import (
+            apply_daily_trail_recommend_from_optuna_json,
+        )
+        apply_daily_trail_recommend_from_optuna_json(
+            getattr(study, "_kis_export_path", None),
+            strategy="us_momentum",
+            log=logger,
+        )
+    except Exception as exc:
+        logger.warning("⚠️ us_momentum 다단트레일 추천 반영 스킵: %s", exc)
+    logger.info(
+        "🚀 [Optuna apply-best] us_momentum gated trial #%d → config_us_momentum",
+        trial.number,
+    )
     return True