#!/usr/bin/env python3 """kis_trader/backtest/optuna_scalping.py — 스캘핑(Reversal) Optuna (Grid add-on).""" from __future__ import annotations import json import logging import os import time from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, List, Optional import optuna from optuna.samplers import RandomSampler, TPESampler from database import TradeDB from kis_trader.backtest import scalping_backtest_common as sbc from kis_trader.backtest.breakout_tick_loader import ( load_breakout_ticks_by_code, tick_coverage_stats, ) from kis_trader.backtest.optuna_search_space import scalp_grid_axis_keys, suggest_scalp_params from kis_trader.backtest.optuna_scalping_tpe_space import ( scalp_tpe_axis_keys, suggest_scalp_params_tpe, ) from kis_trader.backtest.optuna_mode_combo import enrich_out_data_with_mode_combo from kis_trader.backtest.optuna_common import ( announce_optuna_json_path, build_optuna_result_tiers, pick_gated_apply_trial, release_shared_tick_store, set_optuna_trial_stability_attrs, stability_fields_from_trial_attrs, ) from kis_trader.backtest.param_search_cli_common import ( apply_session_to_fixed, combo_passes_search_filters, format_session_hm, ) from kis_trader.backtest.param_search_scalping import ( SCALP_GRID_AXIS_HINTS_KO, _fixed_defaults, _load_candles_for_search, _scalp_grids, _ui_to_engine_params, apply_params_to_db, evaluate_scalp_param_combo, ) from kis_trader.backtest.tail_param_search import _results_dir_for_write from kis_trader.engine import scalping_engine as se from kis_trader.engine.indicator_cache import attach_indicator_caches_to_params from kis_trader.utils.env import get_env_bool, get_env_float logger = logging.getLogger("param_search_optuna") _FAIL_OBJECTIVE = -1e18 @dataclass class ScalpSearchContext: start: str end: str mode: str base_fixed: Dict[str, Any] codes_candles: Dict[str, List[Dict]] universe_by_slot: Optional[Dict[str, List[str]]] universe_source: str fee_rate: float sell_tax: float slot_money: float max_stocks: int total_budget_krw: float period_days: int portfolio: Dict[str, Any] grid_keys: List[str] start_key: str = "" end_key: str = "" ticks_by_code: Any = None tick_rows: int = 0 tick_backtest_meta: Dict[str, Any] = field(default_factory=dict) cache_holder: Dict[str, Any] = field(default_factory=dict) shared_tick_store: Any = None # ws_ticks 공유메모리 핸들 (종료 시 unlink) orderbook_by_code: Dict[str, Any] = field(default_factory=dict) program_by_code: Dict[str, Any] = field(default_factory=dict) orderbook_filter: str = "off" def prepare_scalp_search_context( start: str, end: str, mode: str, *, use_fallback_universe: bool = False, time_start_hm: Optional[int] = None, time_end_hm: Optional[int] = None, slot_money: Optional[float] = None, max_stocks: Optional[int] = None, total_budget_krw: Optional[float] = None, orderbook_filter: str = "off", history_source: Optional[str] = None, ) -> Optional[ScalpSearchContext]: grids = _scalp_grids() if mode == "tpe": grid_axes: Dict[str, Any] = {} logger.info( "📌 mode=tpe — 연속(float/int) 탐색 (Grid categorical 미사용, TPE 가 구간 축소)" ) elif mode not in grids: logger.error( "❌ 스캘핑 mode: %s (fast/trigger/exit/coarse/fine/full/wide/tpe)", mode, ) return None else: grid_axes = grids[mode] base_fixed = _fixed_defaults() if mode == "tpe": base_fixed["skip_hts_scan_dupes"] = False apply_session_to_fixed(base_fixed, time_start_hm=time_start_hm, time_end_hm=time_end_hm) _ob_mode = (orderbook_filter or "off").strip().lower() if _ob_mode == "off": base_fixed["_orderbook_filter_enabled"] = False elif _ob_mode == "on": base_fixed["_orderbook_filter_enabled"] = True ob_filter_on = bool(base_fixed.get("_orderbook_filter_enabled")) or _ob_mode == "auto" logger.info( "📌 호가필터: %s (%s)", _ob_mode.upper(), "적용" if ob_filter_on else "스킵 — 코어 파라미터 순수 탐색 (실매 ORDERBOOK도 OFF 권장 정합)", ) db = TradeDB() try: from kis_trader.backtest.backtest_portfolio_common import load_portfolio_env_row env_row = load_portfolio_env_row(db) finally: db.close() fee_rate, sell_tax, slot_from_env = sbc.fee_and_slot_from_env(env_row, strategy="SCALP") portfolio = sbc.resolve_scalp_portfolio_params( env_row, None, strategy="SCALP", slot_money=slot_money if slot_money is not None else slot_from_env, max_stocks=max_stocks, total_budget_krw=total_budget_krw, ) slot_money_f = float(portfolio["slot_money"]) max_stocks_i = int(portfolio["max_stocks"]) total_budget_f = float(portfolio["total_budget_krw"]) period_days = max( 1, (datetime.strptime(end, "%Y-%m-%d") - datetime.strptime(start, "%Y-%m-%d")).days + 1, ) logger.info( "💼 포트폴리오: 1회 %s원 | 동시 %d종 | 총한도 %s원 | 매매 %s", f"{slot_money_f:,.0f}", max_stocks_i, f"{total_budget_f:,.0f}", format_session_hm(base_fixed), ) # grid_axes 는 상단에서 mode별 설정 (tpe=빈 dict). grids[mode] 재조회 금지. rsi_cands = grid_axes.get("rsi_period") or [base_fixed.get("rsi_period") or 7] try: rsi_period = max(int(float(x)) for x in rsi_cands) except (TypeError, ValueError): rsi_period = int(base_fixed.get("rsi_period") or 7) if mode == "tpe": # TPE 상한까지 RSI 로드 (3~14) rsi_period = max(rsi_period, 14) from kis_trader.backtest.universe_history_source import ( resolve_backtest_universe_history_source, ) _hs = resolve_backtest_universe_history_source(history_source) base_fixed["_universe_history_source"] = _hs codes_candles = _load_candles_for_search( start, end, rsi_period, history_source=_hs, ) if not codes_candles: logger.error("❌ 캔들 데이터 없음") return None logger.info("✅ 데이터 로드: %s종목 (history=%s)", len(codes_candles), _hs) # ── 틱재생 — history_source=ls 이면 ls_ws_ticks ── start_key = (start.replace("-", "") + "0000") if start else "202601010000" end_key = (end.replace("-", "") + "2359") if end else "999912312359" ticks_by_code: Dict[str, Any] = {} tick_rows = 0 tick_backtest_meta: Dict[str, Any] = {} engine_probe = _ui_to_engine_params(base_fixed) engine_probe["backtest_tick_fallback_ohlc"] = False base_fixed["backtest_tick_fallback_ohlc"] = False if sbc._scalp_backtest_wants_ticks(engine_probe): _tick_db = TradeDB() try: if _hs in ("ls", "ls_condition", "ls_ws"): from kis_trader.backtest.ls_history_loaders import load_ls_ticks_by_code ticks_by_code, tick_rows = load_ls_ticks_by_code( _tick_db, start_key, end_key, set(codes_candles.keys()), ) _tick_lbl = "ls_ws_ticks" else: ticks_by_code, tick_rows = load_breakout_ticks_by_code( _tick_db, start_key, end_key, set(codes_candles.keys()), ) _tick_lbl = "ws_ticks" tick_backtest_meta = tick_coverage_stats(codes_candles, ticks_by_code) tick_backtest_meta["ws_tick_rows_loaded"] = tick_rows tick_backtest_meta["tick_table"] = _tick_lbl cov = tick_backtest_meta.get("tick_bar_coverage_pct", 0) logger.info( "✅ %s %s건 | 분봉 커버리지 %s%% (%s/%s종목)", _tick_lbl, f"{tick_rows:,}", cov, tick_backtest_meta.get("tick_codes_with_data", 0), tick_backtest_meta.get("tick_codes_total", 0), ) if tick_rows <= 0: logger.warning( "⚠️ ws_ticks 없음 — SCALP Optuna 가 OHLC만 사용 " "(틱 수집 후 재탐색, FALLBACK_OHLC 기본 OFF)" ) else: _fb = get_env_bool("SCALP_BACKTEST_TICK_FALLBACK_OHLC", False) logger.info( "📌 틱재생(ws_ticks): ON — OHLC 폴백 %s", "ON" if _fb else "OFF", ) finally: _tick_db.close() shared_tick_store = None if get_env_bool("OPTUNA_PARAM_SEARCH_SHARED_TICKS", True) and ticks_by_code: from kis_trader.backtest.shared_ticks import build_shared_ticks_view _view, shared_tick_store = build_shared_ticks_view(ticks_by_code, enabled=True) if shared_tick_store is not None: import atexit as _atexit _atexit.register(shared_tick_store.unlink) logger.info("📦 ws_ticks 공유메모리 ON (Optuna) — dict 사본 제거, RAM 절감") ticks_by_code = _view import gc as _gc _gc.collect() try: import ctypes as _ctypes _ctypes.CDLL("libc.so.6").malloc_trim(0) except Exception: pass start_ymd = start.replace("-", "") if start else "" end_ymd = end.replace("-", "") if end else "" universe_by_slot = None universe_source = "sim" fallback_sim_interval = 5 if not use_fallback_universe and start_ymd and end_ymd: history, src, n_slots, scan_iv = sbc.resolve_scalp_universe( start_ymd, end_ymd, use_saved_history=True, strategy_id="SCALP", history_source=_hs, ) if history: universe_by_slot = history universe_source = src base_fixed["scan_interval_min"] = scan_iv avg = sum(len(v) for v in history.values()) / max(1, n_slots) logger.info( "✅ 유니버스: SCALP 이력 src=%s | %d슬롯 · 평균 %.1f종목", src, n_slots, avg, ) if universe_by_slot is None: universe_top_n = int(os.environ.get("UPDATE_UNIVERSE_TOP_N", "20")) universe_min_score = float(os.environ.get("UPDATE_UNIVERSE_MIN_SCORE", "4.0")) universe_by_slot = se.build_universe_simulation( codes_candles, top_n=universe_top_n, min_score=universe_min_score, scan_interval_min=fallback_sim_interval, ) base_fixed["scan_interval_min"] = fallback_sim_interval universe_source = "sim" logger.info("📌 유니버스: 개미털기 시뮬 fallback (%d분)", fallback_sim_interval) else: base_fixed["scan_interval_min"] = 1 cache_holder: Dict[str, Any] = {} attach_indicator_caches_to_params(cache_holder, codes_candles) # 호가·프로그램 스냅샷 (필터 ON + 그리드 스윕/재생용) orderbook_by_code: Dict[str, Any] = {} program_by_code: Dict[str, Any] = {} _ob_axes = ("max_spread_pct", "min_bid_ask_ratio", "ask_max_mult") _ob_sweeping = any(len(set(grid_axes.get(k) or [])) > 1 for k in _ob_axes) if ob_filter_on: from kis_trader.backtest.trigger_snapshot_loader import load_trigger_snapshots_by_code _ob_db = TradeDB() try: engine_probe["_orderbook_filter_enabled"] = True orderbook_by_code, program_by_code, trigger_snap_meta = load_trigger_snapshots_by_code( _ob_db, start_key, end_key, set(codes_candles.keys()), engine_params=engine_probe, strategy="SCALP", ) ob_rows = int(trigger_snap_meta.get("ws_orderbook_rows_loaded") or 0) pg_rows = int(trigger_snap_meta.get("ws_program_rows_loaded") or 0) logger.info( "✅ TRIGGER 스냅샷 ws_orderbook %s건 | ws_program %s건%s", f"{ob_rows:,}", f"{pg_rows:,}", " (호가축 스윕)" if _ob_sweeping else "", ) if ob_rows <= 0: logger.warning( "⚠️ ws_orderbook 거의 없음 — 호가필터 ON 이어도 스냅샷 없으면 통과(미차단). " "수집 늘린 뒤 재탐색 권장." ) finally: _ob_db.close() return ScalpSearchContext( start=start, end=end, mode=mode, base_fixed=base_fixed, codes_candles=codes_candles, universe_by_slot=universe_by_slot, universe_source=universe_source, fee_rate=fee_rate, sell_tax=sell_tax, slot_money=slot_money_f, max_stocks=max_stocks_i, total_budget_krw=total_budget_f, period_days=period_days, portfolio=portfolio, grid_keys=scalp_tpe_axis_keys() if mode == "tpe" else scalp_grid_axis_keys(mode), start_key=start_key, end_key=end_key, ticks_by_code=ticks_by_code, tick_rows=int(tick_rows), tick_backtest_meta=tick_backtest_meta, cache_holder=cache_holder, shared_tick_store=shared_tick_store, orderbook_by_code=orderbook_by_code, program_by_code=program_by_code, orderbook_filter=_ob_mode, ) def _make_sampler(name: str, seed: Optional[int]): n = (name or "tpe").strip().lower() if n == "random": return RandomSampler(seed=seed) # multivariate TPE + 조건부 suggest 시 independent sampling 경고가 trial마다 폭주 → 억제 return TPESampler(seed=seed, multivariate=True, warn_independent_sampling=False) def _scalp_objective_value(result: Dict[str, Any], sort_by: str) -> float: pnl = float(result["total_pnl"]) if sort_by == "score": mdd_floor = get_env_float("SCALP_SCORE_MDD_FLOOR", 5000.0) mdd = float(result.get("mdd") or 0) return pnl / max(mdd, mdd_floor) if sort_by == "win_rate": return float(result["win_rate"]) return pnl def run_scalp_optuna( ctx: ScalpSearchContext, *, n_trials: int, storage_url: str, study_name: str, min_trades: int, min_win_rate: float, min_pf: float, sort_by: str = "pnl", sampler_name: str = "tpe", seed: Optional[int] = None, n_jobs: int = 1, show_progress: bool = True, ) -> optuna.Study: study = optuna.create_study( study_name=study_name, storage=storage_url, load_if_exists=True, direction="maximize", sampler=_make_sampler(sampler_name, seed), ) def objective(trial: optuna.Trial) -> float: if ctx.mode == "tpe": combo = suggest_scalp_params_tpe(trial) else: combo = suggest_scalp_params(trial, ctx.mode) result = evaluate_scalp_param_combo( combo, base_fixed=ctx.base_fixed, grid_keys=ctx.grid_keys, codes_candles=ctx.codes_candles, min_trades=min_trades, min_win_rate=min_win_rate, min_pf=min_pf, universe_by_slot=ctx.universe_by_slot, slot_money=ctx.slot_money, max_stocks=ctx.max_stocks, total_budget_krw=ctx.total_budget_krw, fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax, period_days=ctx.period_days, cache_holder=ctx.cache_holder, ticks_by_code=ctx.ticks_by_code, orderbook_by_code=ctx.orderbook_by_code, program_by_code=ctx.program_by_code, start_key=ctx.start_key, end_key=ctx.end_key, ) if result is None: trial.set_user_attr("gates_ok", False) return _FAIL_OBJECTIVE obj = _scalp_objective_value(result, sort_by) trial.set_user_attr("gates_ok", True) trial.set_user_attr("total_pnl", float(result["total_pnl"])) trial.set_user_attr("win_rate", float(result["win_rate"])) trial.set_user_attr("pf", float(result.get("pf") or 0)) trial.set_user_attr("mdd", float(result.get("mdd") or 0)) trial.set_user_attr( "score", float(obj if sort_by == "score" else _scalp_objective_value(result, "score")), ) trial.set_user_attr("total_trades", int(result["total_trades"])) trial.set_user_attr( "merged_json", json.dumps(result.get("merged_params") or {}, ensure_ascii=False), ) set_optuna_trial_stability_attrs(trial, result) return float(obj) logger.info( "🔬 Optuna SCALP | study=%s | mode=%s | trials=%d | sort=%s | universe=%s | ticks=%s", study_name, ctx.mode, n_trials, sort_by, ctx.universe_source, f"{ctx.tick_rows:,}", ) t0 = time.time() try: study.optimize(objective, n_trials=n_trials, n_jobs=n_jobs, show_progress_bar=show_progress) elapsed = time.time() - t0 passing: List[Dict[str, Any]] = [] for trial in study.trials: if trial.state != optuna.trial.TrialState.COMPLETE: continue if not trial.user_attrs.get("gates_ok"): continue merged_raw = trial.user_attrs.get("merged_json") or "{}" try: merged = json.loads(merged_raw) except json.JSONDecodeError: merged = dict(trial.params) row = { "params": dict(trial.params), "merged_params": merged, "total_trades": int(trial.user_attrs.get("total_trades") or 0), "win_rate": float(trial.user_attrs.get("win_rate") or 0), "total_pnl": float(trial.user_attrs.get("total_pnl") or 0), "pf": float(trial.user_attrs.get("pf") or 0), "mdd": float(trial.user_attrs.get("mdd") or 0), "score": float(trial.user_attrs.get("score") or 0), "optuna_trial_number": trial.number, } row.update(stability_fields_from_trial_attrs(trial)) passing.append(row) if sort_by == "score": passing.sort(key=lambda r: (-r["score"], -r["total_pnl"], -r["win_rate"])) elif sort_by == "win_rate": passing.sort(key=lambda r: (-r["win_rate"], -r["total_pnl"])) else: passing.sort(key=lambda r: (-r["total_pnl"], -r["win_rate"])) tiers = build_optuna_result_tiers(passing, sort_by=sort_by) out_data = { "engine": "optuna", "strategy": "scalp", "mode": ctx.mode, "start": ctx.start, "end": ctx.end, "universe_source": ctx.universe_source, "slot_money": int(ctx.slot_money), "max_stocks": ctx.max_stocks, "total_budget_krw": int(ctx.total_budget_krw), "backtest_days": ctx.period_days, "min_trades": min_trades, "min_win_rate": min_win_rate, "min_pf": min_pf, "sort_by": sort_by, "grid_keys": ctx.grid_keys, "grid_axis_hints": { k: SCALP_GRID_AXIS_HINTS_KO[k] for k in ctx.grid_keys if k in SCALP_GRID_AXIS_HINTS_KO }, "optuna_study_name": study_name, "optuna_storage": storage_url, "optuna_n_trials_requested": n_trials, "optuna_trials_completed": len(study.trials), "optuna_best_value": study.best_value if study.best_trial else None, "optuna_best_trial_number": study.best_trial.number if study.best_trial else None, "elapsed_sec": round(elapsed, 1), "ws_tick_rows_loaded": int(ctx.tick_rows), "tick_backtest": ctx.tick_backtest_meta, **tiers, } ts = datetime.now().strftime("%Y%m%d_%H%M%S") out_path = os.path.join(_results_dir_for_write(), f"optuna_scalp_{ctx.mode}_{ts}.json") with open(out_path, "w", encoding="utf-8") as f: json.dump(out_data, f, indent=2, ensure_ascii=False) announce_optuna_json_path( out_path, strategy="scalp", mode=ctx.mode, note="중간저장(mode 전)", log=logger, ) def _eval_mode(combo: Dict[str, Any]) -> Optional[Dict[str, Any]]: return evaluate_scalp_param_combo( combo, base_fixed=ctx.base_fixed, grid_keys=ctx.grid_keys, codes_candles=ctx.codes_candles, min_trades=1, min_win_rate=0.0, min_pf=0.0, universe_by_slot=ctx.universe_by_slot, slot_money=ctx.slot_money, max_stocks=ctx.max_stocks, total_budget_krw=ctx.total_budget_krw, fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax, period_days=ctx.period_days, cache_holder=ctx.cache_holder, ticks_by_code=ctx.ticks_by_code, orderbook_by_code=ctx.orderbook_by_code, program_by_code=ctx.program_by_code, start_key=ctx.start_key, end_key=ctx.end_key, include_trades=True, ) def _save_partial(_data: Dict[str, Any]) -> None: with open(out_path, "w", encoding="utf-8") as f: json.dump(_data, f, indent=2, ensure_ascii=False) announce_optuna_json_path( out_path, strategy="scalp", mode=ctx.mode, note="mode_combo params 저장(실측 전)", log=logger, ) enrich_out_data_with_mode_combo( out_data, evaluate_fn=_eval_mode, grid_keys=ctx.grid_keys, log=logger, on_partial_save=_save_partial, ) with open(out_path, "w", encoding="utf-8") as f: json.dump(out_data, f, indent=2, ensure_ascii=False) announce_optuna_json_path( out_path, strategy="scalp", mode=ctx.mode, note="최종 JSON", log=logger, ) study._kis_export_path = out_path # type: ignore[attr-defined] return study finally: release_shared_tick_store(ctx, log=logger) def apply_best_scalp_trial(study: optuna.Study) -> bool: trial = pick_gated_apply_trial(study, sort_by="score", fail_objective=_FAIL_OBJECTIVE) if trial is None: logger.warning( "⚠️ 사후게이트(results_gated) 통과 trial 없음 — DB 미적용" ) return False pnl = float(trial.user_attrs.get("total_pnl") or 0) if pnl <= 0: logger.warning("⚠️ gated trial 총손익 ≤ 0 — DB 미적용") return False merged_raw = trial.user_attrs.get("merged_json") or "{}" merged = json.loads(merged_raw) apply_params_to_db(merged) logger.info("🚀 [Optuna apply-best] scalp gated trial #%d → env_config", trial.number) try: from kis_trader.backtest.optuna_daily_trail_recommend import ( apply_daily_trail_recommend_from_optuna_json, ) apply_daily_trail_recommend_from_optuna_json( getattr(study, "_kis_export_path", None), strategy="scalp", log=logger, ) except Exception as exc: logger.warning("⚠️ 다단트레일 추천 반영 스킵: %s", exc) return True