500 lines
19 KiB
Python
500 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""전 전략 Optuna JSON 정합 재실측 — DB 미변경.
|
|
|
|
Usage:
|
|
python3 -u scripts/multi_optuna_parity_reeval.py \\
|
|
--strategies scalp,breakout_atr,breakout_fixed,momentum,tail \\
|
|
--out logs/multi_optuna_parity_OUT.json
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
import traceback
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
|
|
DEFAULT_JSON = {
|
|
"scalp": "kis_trader/backtest/results/optuna_scalp_tpe_20260822_001653.json",
|
|
"breakout_atr": "kis_trader/backtest/results/optuna_breakout_tpe_20260821_234943.json",
|
|
"breakout_fixed": "kis_trader/backtest/results/optuna_breakout_tpe_20260821_233130.json",
|
|
"momentum": "kis_trader/backtest/results/optuna_momentum_tpe_20260821_220227.json",
|
|
"tail": "kis_trader/backtest/results/optuna_tail_tpe_20260821_225356.json",
|
|
}
|
|
|
|
# 웹 최근 BT (참고 대조) — 재실측 대상에 web_job_params 포함 시
|
|
WEB_BT = {
|
|
"scalp": "kis_trader/backtest/results/scalp_bt_20260818_20260821_20260822_024250.json",
|
|
"breakout": "kis_trader/backtest/results/breakout_bt_20260818_20260821_20260822_024507.json",
|
|
}
|
|
|
|
|
|
def _find_trial(d: dict, n: int) -> Optional[dict]:
|
|
for key in ("results_all", "results", "results_gated", "results_stable"):
|
|
for x in d.get(key) or []:
|
|
if isinstance(x, dict) and x.get("optuna_trial_number") == n:
|
|
return x
|
|
return None
|
|
|
|
|
|
def _rec(t: dict) -> dict:
|
|
return {
|
|
"total_pnl": float(t.get("total_pnl") or 0),
|
|
"total_trades": int(t.get("total_trades") or 0),
|
|
"win_rate": t.get("win_rate"),
|
|
"pf": t.get("pf"),
|
|
}
|
|
|
|
|
|
def _cmp(label: str, recorded: dict, reeval: Optional[dict]) -> dict:
|
|
if reeval is None:
|
|
print(f" {label}: FAIL no result", flush=True)
|
|
return {
|
|
"label": label,
|
|
"ok": False,
|
|
"recorded": recorded,
|
|
"reeval": None,
|
|
"same": False,
|
|
}
|
|
dp = float(reeval["total_pnl"]) - float(recorded.get("total_pnl") or 0)
|
|
dt = int(reeval["total_trades"]) - int(recorded.get("total_trades") or 0)
|
|
same = abs(dp) < 0.5 and dt == 0
|
|
print(
|
|
f" {label}: recorded PnL={recorded.get('total_pnl')} tr={recorded.get('total_trades')} | "
|
|
f"reeval PnL={reeval['total_pnl']:.0f} tr={reeval['total_trades']} | "
|
|
f"Δpnl={dp:.0f} Δtr={dt} same={same}",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"label": label,
|
|
"ok": True,
|
|
"recorded": recorded,
|
|
"reeval": reeval,
|
|
"delta_pnl": dp,
|
|
"delta_trades": dt,
|
|
"same": same,
|
|
}
|
|
|
|
|
|
def _pack_eval(r: Optional[dict]) -> Optional[dict]:
|
|
if not r:
|
|
return None
|
|
return {
|
|
"total_pnl": float(r["total_pnl"]),
|
|
"total_trades": int(r["total_trades"]),
|
|
"win_rate": float(r.get("win_rate") or 0),
|
|
"pf": float(r.get("pf") or 0),
|
|
}
|
|
|
|
|
|
def run_scalp(d: dict, trials: List[int], include_web: bool) -> Dict[str, Any]:
|
|
from kis_trader.backtest.optuna_scalping import prepare_scalp_search_context
|
|
from kis_trader.backtest.param_search_scalping import evaluate_scalp_param_combo
|
|
|
|
start, end = d["start"], d["end"]
|
|
slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
|
|
grid_keys = list(d.get("grid_keys") or [])
|
|
print(f"[scalp] prepare {start}~{end} OB=off", flush=True)
|
|
ctx = prepare_scalp_search_context(
|
|
start, end, "tpe",
|
|
slot_money=slot, max_stocks=ms, total_budget_krw=bud,
|
|
orderbook_filter="off", history_source="kiwoom",
|
|
)
|
|
if ctx is None:
|
|
return {"strategy": "scalp", "error": "prepare_failed"}
|
|
base = dict(ctx.base_fixed)
|
|
base["_orderbook_filter_enabled"] = False
|
|
comparisons = []
|
|
|
|
def eval_combo(combo: dict) -> Optional[dict]:
|
|
return evaluate_scalp_param_combo(
|
|
combo,
|
|
base_fixed=base,
|
|
grid_keys=grid_keys,
|
|
codes_candles=ctx.codes_candles,
|
|
min_trades=1, min_win_rate=0.0, min_pf=0.0,
|
|
universe_by_slot=ctx.universe_by_slot,
|
|
slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
|
|
total_budget_krw=ctx.total_budget_krw,
|
|
fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
|
|
period_days=ctx.period_days, cache_holder=ctx.cache_holder,
|
|
ticks_by_code=ctx.ticks_by_code,
|
|
orderbook_by_code=ctx.orderbook_by_code,
|
|
program_by_code=ctx.program_by_code,
|
|
start_key=ctx.start_key, end_key=ctx.end_key,
|
|
)
|
|
|
|
for n in trials:
|
|
t = _find_trial(d, n)
|
|
if not t:
|
|
comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
|
|
continue
|
|
print(f"--- scalp #{n} ---", flush=True)
|
|
comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
|
|
|
|
mc = d.get("mode_combo") or {}
|
|
if mc.get("params"):
|
|
bt = mc.get("backtest") or {}
|
|
print("--- scalp mode_combo ---", flush=True)
|
|
comparisons.append(
|
|
_cmp(
|
|
"mode_combo",
|
|
{
|
|
"total_pnl": bt.get("total_pnl"),
|
|
"total_trades": bt.get("total_trades"),
|
|
"win_rate": bt.get("win_rate"),
|
|
"pf": bt.get("pf"),
|
|
},
|
|
_pack_eval(eval_combo(dict(mc["params"]))),
|
|
)
|
|
)
|
|
|
|
if include_web:
|
|
wp = Path(WEB_BT["scalp"])
|
|
if wp.is_file():
|
|
wj = json.loads(wp.read_text(encoding="utf-8"))
|
|
wparams = dict(wj.get("params") or {})
|
|
# Optuna UI 키만 덮어씀 (단위=웹 저장값 그대로)
|
|
combo = {k: wparams[k] for k in grid_keys if k in wparams}
|
|
# web 에만 있는 흔한 키
|
|
for k in ("sl_pct", "tp_pct", "tp_max_pct", "drop_rate", "rsi_oversold", "rsi_overbought",
|
|
"rsi_period", "shoulder_min_high", "shoulder_cut_pct", "vol_mult", "cooldown_min",
|
|
"max_daily", "high_chase_thr", "max_daily_chg", "min_price", "max_loss_krw",
|
|
"min_margin", "use_defense_filters", "require_reversal_candle"):
|
|
if k in wparams:
|
|
combo[k] = wparams[k]
|
|
sm = wj.get("summary") or {}
|
|
print("--- scalp web_job_params ---", flush=True)
|
|
print(f" web combo keys={sorted(combo.keys())}", flush=True)
|
|
print(
|
|
f" web drop/rsi/sl={combo.get('drop_rate')}/{combo.get('rsi_oversold')}/"
|
|
f"{combo.get('rsi_overbought')}/{combo.get('sl_pct')}",
|
|
flush=True,
|
|
)
|
|
comparisons.append(
|
|
_cmp(
|
|
"web_job_params",
|
|
{
|
|
"total_pnl": sm.get("total_pnl"),
|
|
"total_trades": sm.get("total_trades"),
|
|
"win_rate": sm.get("win_rate"),
|
|
"pf": sm.get("profit_factor") or sm.get("pf"),
|
|
},
|
|
_pack_eval(eval_combo(combo)),
|
|
)
|
|
)
|
|
|
|
return {
|
|
"strategy": "scalp",
|
|
"source_json": DEFAULT_JSON["scalp"],
|
|
"comparisons": comparisons,
|
|
"all_same": all(c.get("same") for c in comparisons if c.get("ok")),
|
|
}
|
|
|
|
|
|
def run_breakout(d: dict, *, label: str, sl_mode: str, trials: List[int], include_web: bool) -> Dict[str, Any]:
|
|
from kis_trader.backtest.optuna_breakout import prepare_breakout_search_context
|
|
from kis_trader.backtest.param_search_breakout import evaluate_breakout_param_combo
|
|
|
|
start, end = d["start"], d["end"]
|
|
slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
|
|
grid_keys = list(d.get("grid_keys") or [])
|
|
print(f"[{label}] prepare {start}~{end} sl_mode={sl_mode} OB=off", flush=True)
|
|
ctx = prepare_breakout_search_context(
|
|
start, end, "tpe",
|
|
slot_money=slot, max_stocks=ms, total_budget_krw=bud,
|
|
orderbook_filter="off", history_source="kiwoom", sl_mode=sl_mode,
|
|
)
|
|
if ctx is None:
|
|
return {"strategy": label, "error": "prepare_failed"}
|
|
base = dict(ctx.base_fixed)
|
|
base["_orderbook_filter_enabled"] = False
|
|
comparisons = []
|
|
|
|
def eval_combo(combo: dict) -> Optional[dict]:
|
|
return evaluate_breakout_param_combo(
|
|
combo,
|
|
base_fixed=base,
|
|
grid_keys=grid_keys,
|
|
codes_candles=ctx.codes_candles,
|
|
min_trades=1, min_win_rate=0.0, min_pf=0.0,
|
|
universe_by_slot=ctx.universe_by_slot,
|
|
slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
|
|
total_budget_krw=ctx.total_budget_krw,
|
|
fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
|
|
period_days=ctx.period_days, cache_holder=ctx.cache_holder,
|
|
ticks_by_code=ctx.ticks_by_code,
|
|
orderbook_by_code=ctx.orderbook_by_code,
|
|
program_by_code=ctx.program_by_code,
|
|
log_verdict_by_code=getattr(ctx, "log_verdict_by_code", None),
|
|
share_denom_by_code=getattr(ctx, "share_denom_by_code", None),
|
|
)
|
|
|
|
for n in trials:
|
|
t = _find_trial(d, n)
|
|
if not t:
|
|
comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
|
|
continue
|
|
print(f"--- {label} #{n} ---", flush=True)
|
|
comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
|
|
|
|
mc = d.get("mode_combo") or {}
|
|
if mc.get("params"):
|
|
bt = mc.get("backtest") or {}
|
|
print(f"--- {label} mode_combo ---", flush=True)
|
|
comparisons.append(
|
|
_cmp(
|
|
"mode_combo",
|
|
{
|
|
"total_pnl": bt.get("total_pnl"),
|
|
"total_trades": bt.get("total_trades"),
|
|
"win_rate": bt.get("win_rate"),
|
|
"pf": bt.get("pf"),
|
|
},
|
|
_pack_eval(eval_combo(dict(mc["params"]))),
|
|
)
|
|
)
|
|
|
|
if include_web and sl_mode == "atr":
|
|
wp = Path(WEB_BT["breakout"])
|
|
if wp.is_file():
|
|
wj = json.loads(wp.read_text(encoding="utf-8"))
|
|
# 웹이 atr#57 과 동일하면 그 trial 재실측으로 충분 — web params 키 요약만
|
|
sm = wj.get("summary") or {}
|
|
t57 = _find_trial(d, 57)
|
|
if t57 and abs(float(sm.get("total_pnl") or 0) - float(t57.get("total_pnl") or 0)) < 1:
|
|
print("--- breakout web == atr#57 (skip separate web combo) ---", flush=True)
|
|
comparisons.append(
|
|
{
|
|
"label": "web_matches_atr#57",
|
|
"ok": True,
|
|
"same": True,
|
|
"recorded": _rec(t57),
|
|
"note": "web BT PnL/trades identical to Optuna atr #57",
|
|
}
|
|
)
|
|
|
|
return {
|
|
"strategy": label,
|
|
"sl_mode": sl_mode,
|
|
"source_json": DEFAULT_JSON.get("breakout_atr" if sl_mode == "atr" else "breakout_fixed"),
|
|
"comparisons": comparisons,
|
|
"all_same": all(c.get("same") for c in comparisons if c.get("ok")),
|
|
}
|
|
|
|
|
|
def run_momentum(d: dict, trials: List[int]) -> Dict[str, Any]:
|
|
from kis_trader.backtest.optuna_momentum import prepare_momentum_search_context
|
|
from kis_trader.backtest.param_search_momentum import evaluate_momentum_param_combo
|
|
|
|
start, end = d["start"], d["end"]
|
|
slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
|
|
grid_keys = list(d.get("grid_keys") or [])
|
|
print(f"[momentum] prepare {start}~{end} OB=off", flush=True)
|
|
ctx = prepare_momentum_search_context(
|
|
start, end, "tpe",
|
|
slot_money=slot, max_stocks=ms, total_budget_krw=bud,
|
|
orderbook_filter="off", market="KR",
|
|
)
|
|
if ctx is None:
|
|
return {"strategy": "momentum", "error": "prepare_failed"}
|
|
base = dict(ctx.base_fixed)
|
|
base["_orderbook_filter_enabled"] = False
|
|
comparisons = []
|
|
|
|
def eval_combo(combo: dict) -> Optional[dict]:
|
|
return evaluate_momentum_param_combo(
|
|
combo,
|
|
base_fixed=base,
|
|
grid_keys=grid_keys,
|
|
codes_candles=ctx.codes_candles,
|
|
min_trades=1, min_win_rate=0.0, min_pf=0.0,
|
|
universe_by_slot=ctx.universe_by_slot,
|
|
slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
|
|
total_budget_krw=ctx.total_budget_krw,
|
|
fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
|
|
period_days=ctx.period_days, cache_holder=ctx.cache_holder,
|
|
ticks_by_code=ctx.ticks_by_code,
|
|
orderbook_by_code=ctx.orderbook_by_code,
|
|
program_by_code=getattr(ctx, "program_by_code", None),
|
|
start_key=ctx.start_key,
|
|
end_key=ctx.end_key,
|
|
)
|
|
|
|
for n in trials:
|
|
t = _find_trial(d, n)
|
|
if not t:
|
|
comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
|
|
continue
|
|
print(f"--- momentum #{n} ---", flush=True)
|
|
comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
|
|
|
|
mc = d.get("mode_combo") or {}
|
|
if mc.get("params"):
|
|
bt = mc.get("backtest") or {}
|
|
print("--- momentum mode_combo ---", flush=True)
|
|
comparisons.append(
|
|
_cmp(
|
|
"mode_combo",
|
|
{
|
|
"total_pnl": bt.get("total_pnl"),
|
|
"total_trades": bt.get("total_trades"),
|
|
"win_rate": bt.get("win_rate"),
|
|
"pf": bt.get("pf"),
|
|
},
|
|
_pack_eval(eval_combo(dict(mc["params"]))),
|
|
)
|
|
)
|
|
|
|
return {
|
|
"strategy": "momentum",
|
|
"source_json": DEFAULT_JSON["momentum"],
|
|
"comparisons": comparisons,
|
|
"all_same": all(c.get("same") for c in comparisons if c.get("ok")),
|
|
"note": "과거 #199 JSON vs 재실측 갭은 08-21 일자 — 본 스크립트는 현재 엔진 재현 여부",
|
|
}
|
|
|
|
|
|
def run_tail(d: dict, trials: List[int]) -> Dict[str, Any]:
|
|
from kis_trader.backtest.param_search_optuna import prepare_tail_search_context
|
|
from kis_trader.backtest.tail_param_search import evaluate_tail_param_combo
|
|
|
|
start, end = d["start"], d["end"]
|
|
slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
|
|
tf = int(d.get("timeframe") or 3)
|
|
print(f"[tail] prepare {start}~{end} OB=off", flush=True)
|
|
ctx = prepare_tail_search_context(
|
|
start, end, "tpe", timeframe=tf,
|
|
slot_money=slot, max_stocks=ms, total_budget_krw=bud,
|
|
orderbook_filter="off", history_source="kiwoom", entry_mode="align",
|
|
)
|
|
if ctx is None:
|
|
return {"strategy": "tail", "error": "prepare_failed"}
|
|
base = dict(ctx.base_params)
|
|
base["_orderbook_filter_enabled"] = False
|
|
comparisons = []
|
|
|
|
def eval_combo(combo: dict) -> Optional[dict]:
|
|
return evaluate_tail_param_combo(
|
|
combo,
|
|
base_params=base,
|
|
candles_by_code=ctx.candles_by_code,
|
|
fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
|
|
min_trades=1, min_win_rate=0.0, min_pf=0.0,
|
|
universe_by_slot=ctx.universe_by_slot,
|
|
slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
|
|
total_budget_krw=ctx.total_budget_krw,
|
|
period_days=ctx.period_days, cache_holder=ctx.cache_holder,
|
|
ticks_by_code=ctx.ticks_by_code,
|
|
orderbook_by_code=ctx.orderbook_by_code,
|
|
program_by_code=ctx.program_by_code,
|
|
log_verdict_by_code=ctx.log_verdict_by_code,
|
|
)
|
|
|
|
for n in trials:
|
|
t = _find_trial(d, n)
|
|
if not t:
|
|
comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
|
|
continue
|
|
print(f"--- tail #{n} ---", flush=True)
|
|
comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
|
|
|
|
mc = d.get("mode_combo") or {}
|
|
if mc.get("params"):
|
|
bt = mc.get("backtest") or {}
|
|
print("--- tail mode_combo ---", flush=True)
|
|
comparisons.append(
|
|
_cmp(
|
|
"mode_combo",
|
|
{
|
|
"total_pnl": bt.get("total_pnl"),
|
|
"total_trades": bt.get("total_trades"),
|
|
"win_rate": bt.get("win_rate"),
|
|
"pf": bt.get("pf"),
|
|
},
|
|
_pack_eval(eval_combo(dict(mc["params"]))),
|
|
)
|
|
)
|
|
|
|
return {
|
|
"strategy": "tail",
|
|
"source_json": DEFAULT_JSON["tail"],
|
|
"comparisons": comparisons,
|
|
"all_same": all(c.get("same") for c in comparisons if c.get("ok")),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument(
|
|
"--strategies",
|
|
default="scalp,breakout_atr,breakout_fixed,momentum,tail",
|
|
)
|
|
ap.add_argument("--out", required=True)
|
|
ap.add_argument("--skip-web", action="store_true")
|
|
args = ap.parse_args()
|
|
t0 = time.time()
|
|
include_web = not args.skip_web
|
|
wanted = [s.strip() for s in args.strategies.split(",") if s.strip()]
|
|
results: List[Dict[str, Any]] = []
|
|
|
|
try:
|
|
for name in wanted:
|
|
path = DEFAULT_JSON.get(name)
|
|
if not path or not Path(path).is_file():
|
|
results.append({"strategy": name, "error": f"missing_json:{path}"})
|
|
continue
|
|
d = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
best_n = int(d.get("optuna_best_trial_number") or 0)
|
|
if name == "scalp":
|
|
results.append(run_scalp(d, [best_n], include_web=include_web))
|
|
elif name == "breakout_atr":
|
|
# best + web-matching #57
|
|
trials = [best_n]
|
|
if _find_trial(d, 57):
|
|
trials.append(57)
|
|
results.append(
|
|
run_breakout(d, label="breakout_atr", sl_mode="atr", trials=trials, include_web=include_web)
|
|
)
|
|
elif name == "breakout_fixed":
|
|
results.append(
|
|
run_breakout(d, label="breakout_fixed", sl_mode="fixed", trials=[best_n], include_web=False)
|
|
)
|
|
elif name == "momentum":
|
|
results.append(run_momentum(d, [best_n]))
|
|
elif name == "tail":
|
|
# 이미 검증됨 — best+179
|
|
trials = [best_n]
|
|
if _find_trial(d, 179):
|
|
trials.append(179)
|
|
results.append(run_tail(d, trials))
|
|
else:
|
|
results.append({"strategy": name, "error": "unknown"})
|
|
except Exception:
|
|
traceback.print_exc()
|
|
return 1
|
|
|
|
payload = {
|
|
"elapsed_sec": round(time.time() - t0, 1),
|
|
"orderbook_filter": "off",
|
|
"db_modified": False,
|
|
"strategies": results,
|
|
"all_same": all(r.get("all_same") for r in results if "comparisons" in r),
|
|
}
|
|
out = Path(args.out)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(f"wrote {out} all_same={payload['all_same']} elapsed={payload['elapsed_sec']}s", flush=True)
|
|
for r in results:
|
|
print(
|
|
f" SUMMARY {r.get('strategy')}: all_same={r.get('all_same')} err={r.get('error')}",
|
|
flush=True,
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|