152bf40a4c
NameError 수정 (내 직전 커밋 56b254b 회귀 버그)
- optuna_scalping/breakout/momentum: 함수 내부의 중복 import json 제거.
Python 스코프 규칙: 함수 안에 import 가 있으면 컴파일러가 그 이름을 로컬 변수로
취급 → if False 경로 진입 시 objective/후단에서 UnboundLocalError.
파일 상단 import json 을 그대로 사용.
파케이 초고속 로더: 엔진 선택과 분리 (A안 · 사용자 승인)
- breakout_tick_loader.py:277: BACKTEST_USE_RUST=='1' 게이트 제거.
파케이 파일 존재하면 무조건 Rust I/O 로드 → Python/Rust 어느 엔진이든 초고속.
실패 시 자동 DB 폴백(기존 try/except 유지).
- 정합성 안전: 파케이 = ws_ticks 필터 없는 스냅샷 → DB 로드와 동일. 300건 문제와 무관.
웹 재시작 200. 다음 옵투나 Python 라디오 실행이 이 커밋으로 정상 진행 가능.
Co-authored-by: Cursor <cursoragent@cursor.com>
670 lines
26 KiB
Python
670 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""kis_trader/backtest/optuna_scalping.py — 스캘핑(Reversal) Optuna (Grid add-on)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import optuna
|
|
from optuna.samplers import RandomSampler, TPESampler
|
|
|
|
from database import TradeDB
|
|
from kis_trader.backtest import scalping_backtest_common as sbc
|
|
from kis_trader.backtest.breakout_tick_loader import (
|
|
load_common_ticks_by_code,
|
|
tick_coverage_stats,
|
|
)
|
|
from kis_trader.backtest.optuna_search_space import scalp_grid_axis_keys, suggest_scalp_params
|
|
from kis_trader.backtest.optuna_scalping_tpe_space import (
|
|
scalp_tpe_axis_keys,
|
|
suggest_scalp_params_tpe,
|
|
)
|
|
from kis_trader.backtest.optuna_mode_combo import enrich_out_data_with_mode_combo
|
|
from kis_trader.backtest.optuna_common import (
|
|
announce_optuna_json_path,
|
|
build_optuna_result_tiers,
|
|
pick_gated_apply_trial,
|
|
release_shared_tick_store,
|
|
set_optuna_trial_stability_attrs,
|
|
stability_fields_from_trial_attrs,
|
|
)
|
|
from kis_trader.backtest.param_search_cli_common import (
|
|
apply_session_to_fixed,
|
|
combo_passes_search_filters,
|
|
format_session_hm,
|
|
)
|
|
from kis_trader.backtest.param_search_scalping import (
|
|
SCALP_GRID_AXIS_HINTS_KO,
|
|
_fixed_defaults,
|
|
_load_candles_for_search,
|
|
_scalp_grids,
|
|
_ui_to_engine_params,
|
|
apply_params_to_db,
|
|
evaluate_scalp_param_combo,
|
|
)
|
|
from kis_trader.backtest.tail_param_search import _results_dir_for_write
|
|
from kis_trader.engine import scalping_engine as se
|
|
from kis_trader.engine.indicator_cache import attach_indicator_caches_to_params
|
|
from kis_trader.utils.env import get_env_bool, get_env_float
|
|
|
|
logger = logging.getLogger("param_search_optuna")
|
|
|
|
_FAIL_OBJECTIVE = -1e18
|
|
|
|
|
|
@dataclass
|
|
class ScalpSearchContext:
|
|
start: str
|
|
end: str
|
|
mode: str
|
|
base_fixed: Dict[str, Any]
|
|
codes_candles: Dict[str, List[Dict]]
|
|
universe_by_slot: Optional[Dict[str, List[str]]]
|
|
universe_source: str
|
|
fee_rate: float
|
|
sell_tax: float
|
|
slot_money: float
|
|
max_stocks: int
|
|
total_budget_krw: float
|
|
period_days: int
|
|
portfolio: Dict[str, Any]
|
|
grid_keys: List[str]
|
|
start_key: str = ""
|
|
end_key: str = ""
|
|
ticks_by_code: Any = None
|
|
tick_rows: int = 0
|
|
tick_backtest_meta: Dict[str, Any] = field(default_factory=dict)
|
|
cache_holder: Dict[str, Any] = field(default_factory=dict)
|
|
shared_tick_store: Any = None # ws_ticks 공유메모리 핸들 (종료 시 unlink)
|
|
orderbook_by_code: Dict[str, Any] = field(default_factory=dict)
|
|
program_by_code: Dict[str, Any] = field(default_factory=dict)
|
|
orderbook_filter: str = "off"
|
|
|
|
|
|
def prepare_scalp_search_context(
|
|
start: str,
|
|
end: str,
|
|
mode: str,
|
|
*,
|
|
use_fallback_universe: bool = False,
|
|
time_start_hm: Optional[int] = None,
|
|
time_end_hm: Optional[int] = None,
|
|
slot_money: Optional[float] = None,
|
|
max_stocks: Optional[int] = None,
|
|
total_budget_krw: Optional[float] = None,
|
|
orderbook_filter: str = "off",
|
|
history_source: Optional[str] = None,
|
|
) -> Optional[ScalpSearchContext]:
|
|
grids = _scalp_grids()
|
|
if mode == "tpe":
|
|
grid_axes: Dict[str, Any] = {}
|
|
logger.info(
|
|
"📌 mode=tpe — 연속(float/int) 탐색 (Grid categorical 미사용, TPE 가 구간 축소)"
|
|
)
|
|
elif mode not in grids:
|
|
logger.error(
|
|
"❌ 스캘핑 mode: %s (fast/trigger/exit/coarse/fine/full/wide/tpe)", mode,
|
|
)
|
|
return None
|
|
else:
|
|
grid_axes = grids[mode]
|
|
|
|
base_fixed = _fixed_defaults()
|
|
if os.environ.get("BACKTEST_USE_RUST") == "1":
|
|
base_fixed["use_rust"] = True
|
|
|
|
if mode == "tpe":
|
|
base_fixed["skip_hts_scan_dupes"] = False
|
|
apply_session_to_fixed(base_fixed, time_start_hm=time_start_hm, time_end_hm=time_end_hm)
|
|
|
|
_ob_mode = (orderbook_filter or "off").strip().lower()
|
|
if _ob_mode == "off":
|
|
base_fixed["_orderbook_filter_enabled"] = False
|
|
elif _ob_mode == "on":
|
|
base_fixed["_orderbook_filter_enabled"] = True
|
|
from kis_trader.backtest.optuna_tpe_common import optuna_tpe_needs_orderbook_feed
|
|
|
|
# TPE+호가축: CLI off 여도 스냅 로드. trial 이 _orderbook_filter_enabled 로 ON/OFF.
|
|
need_ob_feed = optuna_tpe_needs_orderbook_feed(mode, _ob_mode)
|
|
ob_filter_on = bool(base_fixed.get("_orderbook_filter_enabled")) or _ob_mode == "auto" or need_ob_feed
|
|
logger.info(
|
|
"📌 호가필터: %s (%s)%s",
|
|
_ob_mode.upper(),
|
|
"스냅로드" if need_ob_feed else ("적용" if ob_filter_on else "스킵 — 코어 파라미터 순수 탐색"),
|
|
" · TPE 호가축" if (mode == "tpe" and need_ob_feed) else "",
|
|
)
|
|
|
|
db = TradeDB()
|
|
try:
|
|
from kis_trader.backtest.backtest_portfolio_common import load_portfolio_env_row
|
|
env_row = load_portfolio_env_row(db)
|
|
finally:
|
|
db.close()
|
|
|
|
fee_rate, sell_tax, slot_from_env = sbc.fee_and_slot_from_env(env_row, strategy="SCALP")
|
|
portfolio = sbc.resolve_scalp_portfolio_params(
|
|
env_row, None, strategy="SCALP",
|
|
slot_money=slot_money if slot_money is not None else slot_from_env,
|
|
max_stocks=max_stocks,
|
|
total_budget_krw=total_budget_krw,
|
|
)
|
|
slot_money_f = float(portfolio["slot_money"])
|
|
max_stocks_i = int(portfolio["max_stocks"])
|
|
total_budget_f = float(portfolio["total_budget_krw"])
|
|
period_days = max(
|
|
1,
|
|
(datetime.strptime(end, "%Y-%m-%d") - datetime.strptime(start, "%Y-%m-%d")).days + 1,
|
|
)
|
|
logger.info(
|
|
"💼 포트폴리오: 1회 %s원 | 동시 %d종 | 총한도 %s원 | 매매 %s",
|
|
f"{slot_money_f:,.0f}", max_stocks_i, f"{total_budget_f:,.0f}",
|
|
format_session_hm(base_fixed),
|
|
)
|
|
|
|
# grid_axes 는 상단에서 mode별 설정 (tpe=빈 dict). grids[mode] 재조회 금지.
|
|
rsi_cands = grid_axes.get("rsi_period") or [base_fixed.get("rsi_period") or 7]
|
|
try:
|
|
rsi_period = max(int(float(x)) for x in rsi_cands)
|
|
except (TypeError, ValueError):
|
|
rsi_period = int(base_fixed.get("rsi_period") or 7)
|
|
if mode == "tpe":
|
|
# TPE 상한까지 RSI 로드 (3~14)
|
|
rsi_period = max(rsi_period, 14)
|
|
|
|
from kis_trader.backtest.universe_history_source import (
|
|
resolve_backtest_universe_history_source,
|
|
)
|
|
|
|
_hs = resolve_backtest_universe_history_source(history_source)
|
|
base_fixed["_universe_history_source"] = _hs
|
|
|
|
try:
|
|
from kis_trader.backtest.optuna_feed_trace import log_bt_feed_chain_banner
|
|
|
|
log_bt_feed_chain_banner(context="Optuna-SCALP")
|
|
except Exception:
|
|
pass
|
|
codes_candles = _load_candles_for_search(
|
|
start, end, rsi_period, history_source=_hs,
|
|
)
|
|
if not codes_candles:
|
|
logger.error("❌ 캔들 데이터 없음")
|
|
return None
|
|
logger.info("✅ 데이터 로드: %s종목 (history=%s)", len(codes_candles), _hs)
|
|
|
|
# ── 틱재생 — history_source=ls 이면 ls_ws_ticks ──
|
|
start_key = (start.replace("-", "") + "0000") if start else "202601010000"
|
|
end_key = (end.replace("-", "") + "2359") if end else "999912312359"
|
|
ticks_by_code: Dict[str, Any] = {}
|
|
tick_rows = 0
|
|
tick_backtest_meta: Dict[str, Any] = {}
|
|
engine_probe = _ui_to_engine_params(base_fixed)
|
|
engine_probe["backtest_tick_fallback_ohlc"] = False
|
|
base_fixed["backtest_tick_fallback_ohlc"] = False
|
|
if sbc._scalp_backtest_wants_ticks(engine_probe):
|
|
_tick_db = TradeDB()
|
|
try:
|
|
if _hs in ("ls", "ls_condition", "ls_ws"):
|
|
from kis_trader.backtest.ls_history_loaders import load_ls_ticks_by_code
|
|
|
|
ticks_by_code, tick_rows = load_ls_ticks_by_code(
|
|
_tick_db, start_key, end_key, set(codes_candles.keys()),
|
|
)
|
|
_tick_lbl = "ls_ws_ticks"
|
|
else:
|
|
ticks_by_code, tick_rows = load_common_ticks_by_code(
|
|
_tick_db, start_key, end_key, set(codes_candles.keys()),
|
|
)
|
|
_tick_lbl = "ws_ticks"
|
|
tick_backtest_meta = tick_coverage_stats(codes_candles, ticks_by_code)
|
|
tick_backtest_meta["ws_tick_rows_loaded"] = tick_rows
|
|
tick_backtest_meta["tick_table"] = _tick_lbl
|
|
cov = tick_backtest_meta.get("tick_bar_coverage_pct", 0)
|
|
logger.info(
|
|
"✅ %s %s건 | 분봉 커버리지 %s%% (%s/%s종목)",
|
|
_tick_lbl,
|
|
f"{tick_rows:,}",
|
|
cov,
|
|
tick_backtest_meta.get("tick_codes_with_data", 0),
|
|
tick_backtest_meta.get("tick_codes_total", 0),
|
|
)
|
|
if tick_rows <= 0:
|
|
logger.warning(
|
|
"⚠️ ws_ticks 없음 — SCALP Optuna 가 OHLC만 사용 "
|
|
"(틱 수집 후 재탐색, FALLBACK_OHLC 기본 OFF)"
|
|
)
|
|
else:
|
|
_fb = get_env_bool("SCALP_BACKTEST_TICK_FALLBACK_OHLC", False)
|
|
logger.info(
|
|
"📌 틱재생(ws_ticks): ON — OHLC 폴백 %s",
|
|
"ON" if _fb else "OFF",
|
|
)
|
|
finally:
|
|
_tick_db.close()
|
|
|
|
shared_tick_store = None
|
|
if get_env_bool("OPTUNA_PARAM_SEARCH_SHARED_TICKS", True) and ticks_by_code:
|
|
from kis_trader.backtest.shared_ticks import build_shared_ticks_view
|
|
_view, shared_tick_store = build_shared_ticks_view(ticks_by_code, enabled=True)
|
|
if shared_tick_store is not None:
|
|
import atexit as _atexit
|
|
_atexit.register(shared_tick_store.unlink)
|
|
logger.info("📦 ws_ticks 공유메모리 ON (Optuna) — dict 사본 제거, RAM 절감")
|
|
ticks_by_code = _view
|
|
import gc as _gc
|
|
_gc.collect()
|
|
try:
|
|
import ctypes as _ctypes
|
|
_ctypes.CDLL("libc.so.6").malloc_trim(0)
|
|
except Exception:
|
|
pass
|
|
|
|
start_ymd = start.replace("-", "") if start else ""
|
|
end_ymd = end.replace("-", "") if end else ""
|
|
universe_by_slot = None
|
|
universe_source = "sim"
|
|
fallback_sim_interval = 5
|
|
|
|
if not use_fallback_universe and start_ymd and end_ymd:
|
|
history, src, n_slots, scan_iv = sbc.resolve_scalp_universe(
|
|
start_ymd, end_ymd, use_saved_history=True, strategy_id="SCALP",
|
|
history_source=_hs,
|
|
)
|
|
if history:
|
|
universe_by_slot = history
|
|
universe_source = src
|
|
base_fixed["scan_interval_min"] = scan_iv
|
|
avg = sum(len(v) for v in history.values()) / max(1, n_slots)
|
|
logger.info(
|
|
"✅ 유니버스: SCALP 이력 src=%s | %d슬롯 · 평균 %.1f종목",
|
|
src, n_slots, avg,
|
|
)
|
|
|
|
if universe_by_slot is None:
|
|
universe_top_n = int(os.environ.get("UPDATE_UNIVERSE_TOP_N", "20"))
|
|
universe_min_score = float(os.environ.get("UPDATE_UNIVERSE_MIN_SCORE", "4.0"))
|
|
universe_by_slot = se.build_universe_simulation(
|
|
codes_candles,
|
|
top_n=universe_top_n,
|
|
min_score=universe_min_score,
|
|
scan_interval_min=fallback_sim_interval,
|
|
)
|
|
base_fixed["scan_interval_min"] = fallback_sim_interval
|
|
universe_source = "sim"
|
|
logger.info("📌 유니버스: 개미털기 시뮬 fallback (%d분)", fallback_sim_interval)
|
|
else:
|
|
base_fixed["scan_interval_min"] = 1
|
|
|
|
cache_holder: Dict[str, Any] = {}
|
|
attach_indicator_caches_to_params(cache_holder, codes_candles)
|
|
if ticks_by_code and tick_backtest_meta:
|
|
cache_holder["_tick_meta_cached"] = tick_backtest_meta
|
|
|
|
# 호가·프로그램 스냅샷 (필터 ON + 그리드 스윕/재생용)
|
|
orderbook_by_code: Dict[str, Any] = {}
|
|
program_by_code: Dict[str, Any] = {}
|
|
_ob_axes = ("max_spread_pct", "min_bid_ask_ratio", "ask_max_mult")
|
|
_ob_sweeping = any(len(set(grid_axes.get(k) or [])) > 1 for k in _ob_axes)
|
|
if need_ob_feed or ob_filter_on:
|
|
from kis_trader.backtest.trigger_snapshot_loader import load_trigger_snapshots_by_code
|
|
_ob_db = TradeDB()
|
|
try:
|
|
engine_probe["_orderbook_filter_enabled"] = True
|
|
orderbook_by_code, program_by_code, trigger_snap_meta = load_trigger_snapshots_by_code(
|
|
_ob_db, start_key, end_key, set(codes_candles.keys()),
|
|
engine_params=engine_probe, strategy="SCALP",
|
|
)
|
|
ob_rows = int(trigger_snap_meta.get("ws_orderbook_rows_loaded") or 0)
|
|
pg_rows = int(trigger_snap_meta.get("ws_program_rows_loaded") or 0)
|
|
logger.info(
|
|
"✅ TRIGGER 스냅샷 ws_orderbook %s건 | ws_program %s건%s",
|
|
f"{ob_rows:,}", f"{pg_rows:,}",
|
|
" (호가축 스윕)" if _ob_sweeping else "",
|
|
)
|
|
if ob_rows <= 0:
|
|
logger.warning(
|
|
"⚠️ ws_orderbook 거의 없음 — 호가필터 ON 이어도 스냅샷 없으면 통과(미차단). "
|
|
"수집 늘린 뒤 재탐색 권장."
|
|
)
|
|
finally:
|
|
_ob_db.close()
|
|
|
|
return ScalpSearchContext(
|
|
start=start,
|
|
end=end,
|
|
mode=mode,
|
|
base_fixed=base_fixed,
|
|
codes_candles=codes_candles,
|
|
universe_by_slot=universe_by_slot,
|
|
universe_source=universe_source,
|
|
fee_rate=fee_rate,
|
|
sell_tax=sell_tax,
|
|
slot_money=slot_money_f,
|
|
max_stocks=max_stocks_i,
|
|
total_budget_krw=total_budget_f,
|
|
period_days=period_days,
|
|
portfolio=portfolio,
|
|
grid_keys=scalp_tpe_axis_keys() if mode == "tpe" else scalp_grid_axis_keys(mode),
|
|
start_key=start_key,
|
|
end_key=end_key,
|
|
ticks_by_code=ticks_by_code,
|
|
tick_rows=int(tick_rows),
|
|
tick_backtest_meta=tick_backtest_meta,
|
|
cache_holder=cache_holder,
|
|
shared_tick_store=shared_tick_store,
|
|
orderbook_by_code=orderbook_by_code,
|
|
program_by_code=program_by_code,
|
|
orderbook_filter=_ob_mode,
|
|
)
|
|
|
|
|
|
def _make_sampler(name: str, seed: Optional[int]):
|
|
n = (name or "tpe").strip().lower()
|
|
if n == "random":
|
|
return RandomSampler(seed=seed)
|
|
# multivariate TPE + 조건부 suggest 시 independent sampling 경고가 trial마다 폭주 → 억제
|
|
return TPESampler(seed=seed, multivariate=True, warn_independent_sampling=False)
|
|
|
|
|
|
def _scalp_objective_value(
|
|
result: Dict[str, Any],
|
|
sort_by: str,
|
|
*,
|
|
start: Any = None,
|
|
end: Any = None,
|
|
) -> float:
|
|
from kis_trader.backtest.optuna_common import optuna_objective_value
|
|
return float(optuna_objective_value(
|
|
result, sort_by, start=start, end=end, strategy="scalp",
|
|
))
|
|
|
|
|
|
def run_scalp_optuna(
|
|
ctx: ScalpSearchContext,
|
|
*,
|
|
n_trials: int,
|
|
storage_url: str,
|
|
study_name: str,
|
|
min_trades: int,
|
|
min_win_rate: float,
|
|
min_pf: float,
|
|
sort_by: str = "score",
|
|
sampler_name: str = "tpe",
|
|
seed: Optional[int] = None,
|
|
n_jobs: int = 1,
|
|
show_progress: bool = True,
|
|
) -> optuna.Study:
|
|
study = optuna.create_study(
|
|
study_name=study_name,
|
|
storage=storage_url,
|
|
load_if_exists=True,
|
|
direction="maximize",
|
|
sampler=_make_sampler(sampler_name, seed),
|
|
)
|
|
from kis_trader.backtest.optuna_study_store import (
|
|
bind_study_trials,
|
|
clamp_optimize_n_trials,
|
|
finalize_optuna_export,
|
|
make_study_goal_stop_callback,
|
|
)
|
|
bind_study_trials(study, n_trials=n_trials, log=logger)
|
|
n_trials = clamp_optimize_n_trials(study, n_trials, log=logger)
|
|
|
|
# 2026-09-06 (docs/rust_engine_parity_port_plan.md · 룰 29): Rust 세션은 BACKTEST_USE_RUST=1
|
|
# 일 때만 초기화하고 combo 에 세팅. 이전엔 env 무관하게 항상 세팅 → Python 선택해도 Rust 로 돌던 버그.
|
|
_use_rust_env = os.environ.get("BACKTEST_USE_RUST") == "1"
|
|
rust_session_id: Optional[str] = None
|
|
if _use_rust_env:
|
|
import uuid
|
|
rust_session_id = f"optuna_scalp_{uuid.uuid4().hex[:8]}"
|
|
try:
|
|
import kis_rust_core
|
|
# json 은 파일 상단에서 이미 import. 함수 안 재import 하면 Python 이
|
|
# json 을 로컬 변수로 취급 → if False 경로에서 UnboundLocalError.
|
|
candles_json = json.dumps(ctx.codes_candles)
|
|
kis_rust_core.init_backtest_session_json(
|
|
rust_session_id,
|
|
candles_json
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to init_rust_session: {e}")
|
|
rust_session_id = None
|
|
else:
|
|
logger.info("🐍 [Python 엔진] Rust 세션 미초기화 (BACKTEST_USE_RUST != 1)")
|
|
|
|
def objective(trial: optuna.Trial) -> float:
|
|
if ctx.mode == "tpe":
|
|
combo = suggest_scalp_params_tpe(trial)
|
|
else:
|
|
combo = suggest_scalp_params(trial, ctx.mode)
|
|
|
|
if rust_session_id is not None:
|
|
combo["_rust_session_id"] = rust_session_id
|
|
result = evaluate_scalp_param_combo(
|
|
combo,
|
|
base_fixed=ctx.base_fixed,
|
|
grid_keys=ctx.grid_keys,
|
|
codes_candles=ctx.codes_candles,
|
|
min_trades=min_trades,
|
|
min_win_rate=min_win_rate,
|
|
min_pf=min_pf,
|
|
universe_by_slot=ctx.universe_by_slot,
|
|
slot_money=ctx.slot_money,
|
|
max_stocks=ctx.max_stocks,
|
|
total_budget_krw=ctx.total_budget_krw,
|
|
fee_rate=ctx.fee_rate,
|
|
sell_tax=ctx.sell_tax,
|
|
period_days=ctx.period_days,
|
|
cache_holder=ctx.cache_holder,
|
|
ticks_by_code=ctx.ticks_by_code,
|
|
orderbook_by_code=ctx.orderbook_by_code,
|
|
program_by_code=ctx.program_by_code,
|
|
start_key=ctx.start_key,
|
|
end_key=ctx.end_key,
|
|
)
|
|
if result is None:
|
|
trial.set_user_attr("gates_ok", False)
|
|
return _FAIL_OBJECTIVE
|
|
trial.set_user_attr("gates_ok", True)
|
|
trial.set_user_attr("total_pnl", float(result["total_pnl"]))
|
|
trial.set_user_attr("win_rate", float(result["win_rate"]))
|
|
trial.set_user_attr("pf", float(result.get("pf") or 0))
|
|
trial.set_user_attr("mdd", float(result.get("mdd") or 0))
|
|
trial.set_user_attr("total_trades", int(result["total_trades"]))
|
|
trial.set_user_attr(
|
|
"merged_json",
|
|
json.dumps(result.get("merged_params") or {}, ensure_ascii=False),
|
|
)
|
|
from kis_trader.backtest.optuna_common import optuna_store_trial_score_user_attrs
|
|
set_optuna_trial_stability_attrs(trial, result)
|
|
return optuna_store_trial_score_user_attrs(
|
|
trial, result, sort_by, start=ctx.start, end=ctx.end, strategy="scalp",
|
|
)
|
|
|
|
logger.info(
|
|
"🔬 Optuna SCALP | study=%s | mode=%s | trials=%d | sort=%s | universe=%s | ticks=%s",
|
|
study_name, ctx.mode, n_trials, sort_by, ctx.universe_source,
|
|
f"{ctx.tick_rows:,}",
|
|
)
|
|
t0 = time.time()
|
|
try:
|
|
if n_trials <= 0:
|
|
logger.info("📌 추가 trial 없음 — 기존 study 결과만 정리")
|
|
else:
|
|
study.optimize(
|
|
objective,
|
|
n_trials=n_trials,
|
|
n_jobs=n_jobs,
|
|
show_progress_bar=show_progress,
|
|
callbacks=[make_study_goal_stop_callback(logger)],
|
|
)
|
|
elapsed = time.time() - t0
|
|
|
|
from kis_trader.backtest.optuna_common import optuna_score_fields_from_trial
|
|
passing: List[Dict[str, Any]] = []
|
|
for trial in study.trials:
|
|
if trial.state != optuna.trial.TrialState.COMPLETE:
|
|
continue
|
|
if not trial.user_attrs.get("gates_ok"):
|
|
continue
|
|
merged_raw = trial.user_attrs.get("merged_json") or "{}"
|
|
try:
|
|
merged = json.loads(merged_raw)
|
|
except json.JSONDecodeError:
|
|
merged = dict(trial.params)
|
|
row = {
|
|
"params": dict(trial.params),
|
|
"merged_params": merged,
|
|
"total_trades": int(trial.user_attrs.get("total_trades") or 0),
|
|
"win_rate": float(trial.user_attrs.get("win_rate") or 0),
|
|
"total_pnl": float(trial.user_attrs.get("total_pnl") or 0),
|
|
"pf": float(trial.user_attrs.get("pf") or 0),
|
|
"mdd": float(trial.user_attrs.get("mdd") or 0),
|
|
**optuna_score_fields_from_trial(trial),
|
|
"period_daily_avg_pnl": float(trial.user_attrs.get("period_daily_avg_pnl") or 0),
|
|
"optuna_trial_number": trial.number,
|
|
}
|
|
row.update(stability_fields_from_trial_attrs(trial))
|
|
passing.append(row)
|
|
|
|
from kis_trader.backtest.optuna_common import _sort_optuna_rows
|
|
passing = _sort_optuna_rows(passing, sort_by)
|
|
|
|
tiers = build_optuna_result_tiers(passing, sort_by=sort_by)
|
|
|
|
out_data = {
|
|
"engine": "optuna",
|
|
"strategy": "scalp",
|
|
"mode": ctx.mode,
|
|
"start": ctx.start,
|
|
"end": ctx.end,
|
|
"universe_source": ctx.universe_source,
|
|
"slot_money": int(ctx.slot_money),
|
|
"max_stocks": ctx.max_stocks,
|
|
"total_budget_krw": int(ctx.total_budget_krw),
|
|
"backtest_days": ctx.period_days,
|
|
"min_trades": min_trades,
|
|
"min_win_rate": min_win_rate,
|
|
"min_pf": min_pf,
|
|
"sort_by": sort_by,
|
|
"grid_keys": ctx.grid_keys,
|
|
"grid_axis_hints": {
|
|
k: SCALP_GRID_AXIS_HINTS_KO[k]
|
|
for k in ctx.grid_keys if k in SCALP_GRID_AXIS_HINTS_KO
|
|
},
|
|
"optuna_study_name": study_name,
|
|
"optuna_storage": storage_url,
|
|
"optuna_n_trials_requested": n_trials,
|
|
"optuna_trials_completed": len(study.trials),
|
|
"optuna_best_value": study.best_value if study.best_trial else None,
|
|
"optuna_best_trial_number": study.best_trial.number if study.best_trial else None,
|
|
"elapsed_sec": round(elapsed, 1),
|
|
"ws_tick_rows_loaded": int(ctx.tick_rows),
|
|
"tick_backtest": ctx.tick_backtest_meta,
|
|
**tiers,
|
|
}
|
|
from kis_trader.backtest.optuna_common import annotate_optuna_period_daily_avg
|
|
annotate_optuna_period_daily_avg(out_data)
|
|
|
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
out_path = os.path.join(_results_dir_for_write(), f"optuna_scalp_{ctx.mode}_{ts}.json")
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
json.dump(out_data, f, indent=2, ensure_ascii=False)
|
|
announce_optuna_json_path(
|
|
out_path, strategy="scalp", mode=ctx.mode, note="중간저장(mode 전)", log=logger,
|
|
)
|
|
|
|
def _eval_mode(combo: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
return evaluate_scalp_param_combo(
|
|
combo,
|
|
base_fixed=ctx.base_fixed,
|
|
grid_keys=ctx.grid_keys,
|
|
codes_candles=ctx.codes_candles,
|
|
min_trades=1,
|
|
min_win_rate=0.0,
|
|
min_pf=0.0,
|
|
universe_by_slot=ctx.universe_by_slot,
|
|
slot_money=ctx.slot_money,
|
|
max_stocks=ctx.max_stocks,
|
|
total_budget_krw=ctx.total_budget_krw,
|
|
fee_rate=ctx.fee_rate,
|
|
sell_tax=ctx.sell_tax,
|
|
period_days=ctx.period_days,
|
|
cache_holder=ctx.cache_holder,
|
|
ticks_by_code=ctx.ticks_by_code,
|
|
orderbook_by_code=ctx.orderbook_by_code,
|
|
program_by_code=ctx.program_by_code,
|
|
start_key=ctx.start_key,
|
|
end_key=ctx.end_key,
|
|
include_trades=True,
|
|
)
|
|
|
|
def _save_partial(_data: Dict[str, Any]) -> None:
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
json.dump(_data, f, indent=2, ensure_ascii=False)
|
|
announce_optuna_json_path(
|
|
out_path, strategy="scalp", mode=ctx.mode, note="mode_combo params 저장(실측 전)", log=logger,
|
|
)
|
|
|
|
def _enrich() -> None:
|
|
enrich_out_data_with_mode_combo(
|
|
out_data,
|
|
evaluate_fn=_eval_mode,
|
|
grid_keys=ctx.grid_keys,
|
|
log=logger,
|
|
on_partial_save=_save_partial,
|
|
)
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
json.dump(out_data, f, indent=2, ensure_ascii=False)
|
|
announce_optuna_json_path(
|
|
out_path, strategy="scalp", mode=ctx.mode, note="최종 JSON", log=logger,
|
|
)
|
|
|
|
finalize_optuna_export(
|
|
study,
|
|
out_data=out_data,
|
|
out_path=out_path,
|
|
strategy="scalp",
|
|
mode=ctx.mode,
|
|
enrich_fn=_enrich,
|
|
log=logger,
|
|
)
|
|
study._kis_export_path = out_path # type: ignore[attr-defined]
|
|
return study
|
|
finally:
|
|
release_shared_tick_store(ctx, log=logger)
|
|
|
|
|
|
def apply_best_scalp_trial(study: optuna.Study) -> bool:
|
|
trial = pick_gated_apply_trial(study, sort_by="score", fail_objective=_FAIL_OBJECTIVE)
|
|
if trial is None:
|
|
logger.warning(
|
|
"⚠️ 사후게이트(results_gated) 통과 trial 없음 — DB 미적용"
|
|
)
|
|
return False
|
|
pnl = float(trial.user_attrs.get("total_pnl") or 0)
|
|
if pnl <= 0:
|
|
logger.warning("⚠️ gated trial 총손익 ≤ 0 — DB 미적용")
|
|
return False
|
|
merged_raw = trial.user_attrs.get("merged_json") or "{}"
|
|
merged = json.loads(merged_raw)
|
|
apply_params_to_db(merged)
|
|
logger.info("🚀 [Optuna apply-best] scalp gated trial #%d → env_config", trial.number)
|
|
try:
|
|
from kis_trader.backtest.optuna_daily_trail_recommend import (
|
|
apply_daily_trail_recommend_from_optuna_json,
|
|
)
|
|
apply_daily_trail_recommend_from_optuna_json(
|
|
getattr(study, "_kis_export_path", None),
|
|
strategy="scalp",
|
|
log=logger,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("⚠️ 다단트레일 추천 반영 스킵: %s", exc)
|
|
return True
|