옵투나 8방 후처리를 재탐색으로 변경하기 전전

scripts/multi_optuna_parity_reeval.py

@@ -0,0 +1,499 @@
+#!/usr/bin/env python3
+"""전 전략 Optuna JSON 정합 재실측 — DB 미변경.
+
+Usage:
+  python3 -u scripts/multi_optuna_parity_reeval.py \\
+    --strategies scalp,breakout_atr,breakout_fixed,momentum,tail \\
+    --out logs/multi_optuna_parity_OUT.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import time
+import traceback
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+
+DEFAULT_JSON = {
+    "scalp": "kis_trader/backtest/results/optuna_scalp_tpe_20260822_001653.json",
+    "breakout_atr": "kis_trader/backtest/results/optuna_breakout_tpe_20260821_234943.json",
+    "breakout_fixed": "kis_trader/backtest/results/optuna_breakout_tpe_20260821_233130.json",
+    "momentum": "kis_trader/backtest/results/optuna_momentum_tpe_20260821_220227.json",
+    "tail": "kis_trader/backtest/results/optuna_tail_tpe_20260821_225356.json",
+}
+
+# 웹 최근 BT (참고 대조) — 재실측 대상에 web_job_params 포함 시
+WEB_BT = {
+    "scalp": "kis_trader/backtest/results/scalp_bt_20260818_20260821_20260822_024250.json",
+    "breakout": "kis_trader/backtest/results/breakout_bt_20260818_20260821_20260822_024507.json",
+}
+
+
+def _find_trial(d: dict, n: int) -> Optional[dict]:
+    for key in ("results_all", "results", "results_gated", "results_stable"):
+        for x in d.get(key) or []:
+            if isinstance(x, dict) and x.get("optuna_trial_number") == n:
+                return x
+    return None
+
+
+def _rec(t: dict) -> dict:
+    return {
+        "total_pnl": float(t.get("total_pnl") or 0),
+        "total_trades": int(t.get("total_trades") or 0),
+        "win_rate": t.get("win_rate"),
+        "pf": t.get("pf"),
+    }
+
+
+def _cmp(label: str, recorded: dict, reeval: Optional[dict]) -> dict:
+    if reeval is None:
+        print(f"  {label}: FAIL no result", flush=True)
+        return {
+            "label": label,
+            "ok": False,
+            "recorded": recorded,
+            "reeval": None,
+            "same": False,
+        }
+    dp = float(reeval["total_pnl"]) - float(recorded.get("total_pnl") or 0)
+    dt = int(reeval["total_trades"]) - int(recorded.get("total_trades") or 0)
+    same = abs(dp) < 0.5 and dt == 0
+    print(
+        f"  {label}: recorded PnL={recorded.get('total_pnl')} tr={recorded.get('total_trades')} | "
+        f"reeval PnL={reeval['total_pnl']:.0f} tr={reeval['total_trades']} | "
+        f"Δpnl={dp:.0f} Δtr={dt} same={same}",
+        flush=True,
+    )
+    return {
+        "label": label,
+        "ok": True,
+        "recorded": recorded,
+        "reeval": reeval,
+        "delta_pnl": dp,
+        "delta_trades": dt,
+        "same": same,
+    }
+
+
+def _pack_eval(r: Optional[dict]) -> Optional[dict]:
+    if not r:
+        return None
+    return {
+        "total_pnl": float(r["total_pnl"]),
+        "total_trades": int(r["total_trades"]),
+        "win_rate": float(r.get("win_rate") or 0),
+        "pf": float(r.get("pf") or 0),
+    }
+
+
+def run_scalp(d: dict, trials: List[int], include_web: bool) -> Dict[str, Any]:
+    from kis_trader.backtest.optuna_scalping import prepare_scalp_search_context
+    from kis_trader.backtest.param_search_scalping import evaluate_scalp_param_combo
+
+    start, end = d["start"], d["end"]
+    slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
+    grid_keys = list(d.get("grid_keys") or [])
+    print(f"[scalp] prepare {start}~{end} OB=off", flush=True)
+    ctx = prepare_scalp_search_context(
+        start, end, "tpe",
+        slot_money=slot, max_stocks=ms, total_budget_krw=bud,
+        orderbook_filter="off", history_source="kiwoom",
+    )
+    if ctx is None:
+        return {"strategy": "scalp", "error": "prepare_failed"}
+    base = dict(ctx.base_fixed)
+    base["_orderbook_filter_enabled"] = False
+    comparisons = []
+
+    def eval_combo(combo: dict) -> Optional[dict]:
+        return evaluate_scalp_param_combo(
+            combo,
+            base_fixed=base,
+            grid_keys=grid_keys,
+            codes_candles=ctx.codes_candles,
+            min_trades=1, min_win_rate=0.0, min_pf=0.0,
+            universe_by_slot=ctx.universe_by_slot,
+            slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
+            total_budget_krw=ctx.total_budget_krw,
+            fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
+            period_days=ctx.period_days, cache_holder=ctx.cache_holder,
+            ticks_by_code=ctx.ticks_by_code,
+            orderbook_by_code=ctx.orderbook_by_code,
+            program_by_code=ctx.program_by_code,
+            start_key=ctx.start_key, end_key=ctx.end_key,
+        )
+
+    for n in trials:
+        t = _find_trial(d, n)
+        if not t:
+            comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
+            continue
+        print(f"--- scalp #{n} ---", flush=True)
+        comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
+
+    mc = d.get("mode_combo") or {}
+    if mc.get("params"):
+        bt = mc.get("backtest") or {}
+        print("--- scalp mode_combo ---", flush=True)
+        comparisons.append(
+            _cmp(
+                "mode_combo",
+                {
+                    "total_pnl": bt.get("total_pnl"),
+                    "total_trades": bt.get("total_trades"),
+                    "win_rate": bt.get("win_rate"),
+                    "pf": bt.get("pf"),
+                },
+                _pack_eval(eval_combo(dict(mc["params"]))),
+            )
+        )
+
+    if include_web:
+        wp = Path(WEB_BT["scalp"])
+        if wp.is_file():
+            wj = json.loads(wp.read_text(encoding="utf-8"))
+            wparams = dict(wj.get("params") or {})
+            # Optuna UI 키만 덮어씀 (단위=웹 저장값 그대로)
+            combo = {k: wparams[k] for k in grid_keys if k in wparams}
+            # web 에만 있는 흔한 키
+            for k in ("sl_pct", "tp_pct", "tp_max_pct", "drop_rate", "rsi_oversold", "rsi_overbought",
+                      "rsi_period", "shoulder_min_high", "shoulder_cut_pct", "vol_mult", "cooldown_min",
+                      "max_daily", "high_chase_thr", "max_daily_chg", "min_price", "max_loss_krw",
+                      "min_margin", "use_defense_filters", "require_reversal_candle"):
+                if k in wparams:
+                    combo[k] = wparams[k]
+            sm = wj.get("summary") or {}
+            print("--- scalp web_job_params ---", flush=True)
+            print(f"  web combo keys={sorted(combo.keys())}", flush=True)
+            print(
+                f"  web drop/rsi/sl={combo.get('drop_rate')}/{combo.get('rsi_oversold')}/"
+                f"{combo.get('rsi_overbought')}/{combo.get('sl_pct')}",
+                flush=True,
+            )
+            comparisons.append(
+                _cmp(
+                    "web_job_params",
+                    {
+                        "total_pnl": sm.get("total_pnl"),
+                        "total_trades": sm.get("total_trades"),
+                        "win_rate": sm.get("win_rate"),
+                        "pf": sm.get("profit_factor") or sm.get("pf"),
+                    },
+                    _pack_eval(eval_combo(combo)),
+                )
+            )
+
+    return {
+        "strategy": "scalp",
+        "source_json": DEFAULT_JSON["scalp"],
+        "comparisons": comparisons,
+        "all_same": all(c.get("same") for c in comparisons if c.get("ok")),
+    }
+
+
+def run_breakout(d: dict, *, label: str, sl_mode: str, trials: List[int], include_web: bool) -> Dict[str, Any]:
+    from kis_trader.backtest.optuna_breakout import prepare_breakout_search_context
+    from kis_trader.backtest.param_search_breakout import evaluate_breakout_param_combo
+
+    start, end = d["start"], d["end"]
+    slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
+    grid_keys = list(d.get("grid_keys") or [])
+    print(f"[{label}] prepare {start}~{end} sl_mode={sl_mode} OB=off", flush=True)
+    ctx = prepare_breakout_search_context(
+        start, end, "tpe",
+        slot_money=slot, max_stocks=ms, total_budget_krw=bud,
+        orderbook_filter="off", history_source="kiwoom", sl_mode=sl_mode,
+    )
+    if ctx is None:
+        return {"strategy": label, "error": "prepare_failed"}
+    base = dict(ctx.base_fixed)
+    base["_orderbook_filter_enabled"] = False
+    comparisons = []
+
+    def eval_combo(combo: dict) -> Optional[dict]:
+        return evaluate_breakout_param_combo(
+            combo,
+            base_fixed=base,
+            grid_keys=grid_keys,
+            codes_candles=ctx.codes_candles,
+            min_trades=1, min_win_rate=0.0, min_pf=0.0,
+            universe_by_slot=ctx.universe_by_slot,
+            slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
+            total_budget_krw=ctx.total_budget_krw,
+            fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
+            period_days=ctx.period_days, cache_holder=ctx.cache_holder,
+            ticks_by_code=ctx.ticks_by_code,
+            orderbook_by_code=ctx.orderbook_by_code,
+            program_by_code=ctx.program_by_code,
+            log_verdict_by_code=getattr(ctx, "log_verdict_by_code", None),
+            share_denom_by_code=getattr(ctx, "share_denom_by_code", None),
+        )
+
+    for n in trials:
+        t = _find_trial(d, n)
+        if not t:
+            comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
+            continue
+        print(f"--- {label} #{n} ---", flush=True)
+        comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
+
+    mc = d.get("mode_combo") or {}
+    if mc.get("params"):
+        bt = mc.get("backtest") or {}
+        print(f"--- {label} mode_combo ---", flush=True)
+        comparisons.append(
+            _cmp(
+                "mode_combo",
+                {
+                    "total_pnl": bt.get("total_pnl"),
+                    "total_trades": bt.get("total_trades"),
+                    "win_rate": bt.get("win_rate"),
+                    "pf": bt.get("pf"),
+                },
+                _pack_eval(eval_combo(dict(mc["params"]))),
+            )
+        )
+
+    if include_web and sl_mode == "atr":
+        wp = Path(WEB_BT["breakout"])
+        if wp.is_file():
+            wj = json.loads(wp.read_text(encoding="utf-8"))
+            # 웹이 atr#57 과 동일하면 그 trial 재실측으로 충분 — web params 키 요약만
+            sm = wj.get("summary") or {}
+            t57 = _find_trial(d, 57)
+            if t57 and abs(float(sm.get("total_pnl") or 0) - float(t57.get("total_pnl") or 0)) < 1:
+                print("--- breakout web == atr#57 (skip separate web combo) ---", flush=True)
+                comparisons.append(
+                    {
+                        "label": "web_matches_atr#57",
+                        "ok": True,
+                        "same": True,
+                        "recorded": _rec(t57),
+                        "note": "web BT PnL/trades identical to Optuna atr #57",
+                    }
+                )
+
+    return {
+        "strategy": label,
+        "sl_mode": sl_mode,
+        "source_json": DEFAULT_JSON.get("breakout_atr" if sl_mode == "atr" else "breakout_fixed"),
+        "comparisons": comparisons,
+        "all_same": all(c.get("same") for c in comparisons if c.get("ok")),
+    }
+
+
+def run_momentum(d: dict, trials: List[int]) -> Dict[str, Any]:
+    from kis_trader.backtest.optuna_momentum import prepare_momentum_search_context
+    from kis_trader.backtest.param_search_momentum import evaluate_momentum_param_combo
+
+    start, end = d["start"], d["end"]
+    slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
+    grid_keys = list(d.get("grid_keys") or [])
+    print(f"[momentum] prepare {start}~{end} OB=off", flush=True)
+    ctx = prepare_momentum_search_context(
+        start, end, "tpe",
+        slot_money=slot, max_stocks=ms, total_budget_krw=bud,
+        orderbook_filter="off", market="KR",
+    )
+    if ctx is None:
+        return {"strategy": "momentum", "error": "prepare_failed"}
+    base = dict(ctx.base_fixed)
+    base["_orderbook_filter_enabled"] = False
+    comparisons = []
+
+    def eval_combo(combo: dict) -> Optional[dict]:
+        return evaluate_momentum_param_combo(
+            combo,
+            base_fixed=base,
+            grid_keys=grid_keys,
+            codes_candles=ctx.codes_candles,
+            min_trades=1, min_win_rate=0.0, min_pf=0.0,
+            universe_by_slot=ctx.universe_by_slot,
+            slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
+            total_budget_krw=ctx.total_budget_krw,
+            fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
+            period_days=ctx.period_days, cache_holder=ctx.cache_holder,
+            ticks_by_code=ctx.ticks_by_code,
+            orderbook_by_code=ctx.orderbook_by_code,
+            program_by_code=getattr(ctx, "program_by_code", None),
+            start_key=ctx.start_key,
+            end_key=ctx.end_key,
+        )
+
+    for n in trials:
+        t = _find_trial(d, n)
+        if not t:
+            comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
+            continue
+        print(f"--- momentum #{n} ---", flush=True)
+        comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
+
+    mc = d.get("mode_combo") or {}
+    if mc.get("params"):
+        bt = mc.get("backtest") or {}
+        print("--- momentum mode_combo ---", flush=True)
+        comparisons.append(
+            _cmp(
+                "mode_combo",
+                {
+                    "total_pnl": bt.get("total_pnl"),
+                    "total_trades": bt.get("total_trades"),
+                    "win_rate": bt.get("win_rate"),
+                    "pf": bt.get("pf"),
+                },
+                _pack_eval(eval_combo(dict(mc["params"]))),
+            )
+        )
+
+    return {
+        "strategy": "momentum",
+        "source_json": DEFAULT_JSON["momentum"],
+        "comparisons": comparisons,
+        "all_same": all(c.get("same") for c in comparisons if c.get("ok")),
+        "note": "과거 #199 JSON vs 재실측 갭은 08-21 일자 — 본 스크립트는 현재 엔진 재현 여부",
+    }
+
+
+def run_tail(d: dict, trials: List[int]) -> Dict[str, Any]:
+    from kis_trader.backtest.param_search_optuna import prepare_tail_search_context
+    from kis_trader.backtest.tail_param_search import evaluate_tail_param_combo
+
+    start, end = d["start"], d["end"]
+    slot, ms, bud = float(d["slot_money"]), int(d["max_stocks"]), float(d["total_budget_krw"])
+    tf = int(d.get("timeframe") or 3)
+    print(f"[tail] prepare {start}~{end} OB=off", flush=True)
+    ctx = prepare_tail_search_context(
+        start, end, "tpe", timeframe=tf,
+        slot_money=slot, max_stocks=ms, total_budget_krw=bud,
+        orderbook_filter="off", history_source="kiwoom", entry_mode="align",
+    )
+    if ctx is None:
+        return {"strategy": "tail", "error": "prepare_failed"}
+    base = dict(ctx.base_params)
+    base["_orderbook_filter_enabled"] = False
+    comparisons = []
+
+    def eval_combo(combo: dict) -> Optional[dict]:
+        return evaluate_tail_param_combo(
+            combo,
+            base_params=base,
+            candles_by_code=ctx.candles_by_code,
+            fee_rate=ctx.fee_rate, sell_tax=ctx.sell_tax,
+            min_trades=1, min_win_rate=0.0, min_pf=0.0,
+            universe_by_slot=ctx.universe_by_slot,
+            slot_money=ctx.slot_money, max_stocks=ctx.max_stocks,
+            total_budget_krw=ctx.total_budget_krw,
+            period_days=ctx.period_days, cache_holder=ctx.cache_holder,
+            ticks_by_code=ctx.ticks_by_code,
+            orderbook_by_code=ctx.orderbook_by_code,
+            program_by_code=ctx.program_by_code,
+            log_verdict_by_code=ctx.log_verdict_by_code,
+        )
+
+    for n in trials:
+        t = _find_trial(d, n)
+        if not t:
+            comparisons.append({"label": f"#{n}", "ok": False, "error": "missing"})
+            continue
+        print(f"--- tail #{n} ---", flush=True)
+        comparisons.append(_cmp(f"#{n}", _rec(t), _pack_eval(eval_combo(dict(t.get("params") or {})))))
+
+    mc = d.get("mode_combo") or {}
+    if mc.get("params"):
+        bt = mc.get("backtest") or {}
+        print("--- tail mode_combo ---", flush=True)
+        comparisons.append(
+            _cmp(
+                "mode_combo",
+                {
+                    "total_pnl": bt.get("total_pnl"),
+                    "total_trades": bt.get("total_trades"),
+                    "win_rate": bt.get("win_rate"),
+                    "pf": bt.get("pf"),
+                },
+                _pack_eval(eval_combo(dict(mc["params"]))),
+            )
+        )
+
+    return {
+        "strategy": "tail",
+        "source_json": DEFAULT_JSON["tail"],
+        "comparisons": comparisons,
+        "all_same": all(c.get("same") for c in comparisons if c.get("ok")),
+    }
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "--strategies",
+        default="scalp,breakout_atr,breakout_fixed,momentum,tail",
+    )
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--skip-web", action="store_true")
+    args = ap.parse_args()
+    t0 = time.time()
+    include_web = not args.skip_web
+    wanted = [s.strip() for s in args.strategies.split(",") if s.strip()]
+    results: List[Dict[str, Any]] = []
+
+    try:
+        for name in wanted:
+            path = DEFAULT_JSON.get(name)
+            if not path or not Path(path).is_file():
+                results.append({"strategy": name, "error": f"missing_json:{path}"})
+                continue
+            d = json.loads(Path(path).read_text(encoding="utf-8"))
+            best_n = int(d.get("optuna_best_trial_number") or 0)
+            if name == "scalp":
+                results.append(run_scalp(d, [best_n], include_web=include_web))
+            elif name == "breakout_atr":
+                # best + web-matching #57
+                trials = [best_n]
+                if _find_trial(d, 57):
+                    trials.append(57)
+                results.append(
+                    run_breakout(d, label="breakout_atr", sl_mode="atr", trials=trials, include_web=include_web)
+                )
+            elif name == "breakout_fixed":
+                results.append(
+                    run_breakout(d, label="breakout_fixed", sl_mode="fixed", trials=[best_n], include_web=False)
+                )
+            elif name == "momentum":
+                results.append(run_momentum(d, [best_n]))
+            elif name == "tail":
+                # 이미 검증됨 — best+179
+                trials = [best_n]
+                if _find_trial(d, 179):
+                    trials.append(179)
+                results.append(run_tail(d, trials))
+            else:
+                results.append({"strategy": name, "error": "unknown"})
+    except Exception:
+        traceback.print_exc()
+        return 1
+
+    payload = {
+        "elapsed_sec": round(time.time() - t0, 1),
+        "orderbook_filter": "off",
+        "db_modified": False,
+        "strategies": results,
+        "all_same": all(r.get("all_same") for r in results if "comparisons" in r),
+    }
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(f"wrote {out} all_same={payload['all_same']} elapsed={payload['elapsed_sec']}s", flush=True)
+    for r in results:
+        print(
+            f"  SUMMARY {r.get('strategy')}: all_same={r.get('all_same')} err={r.get('error')}",
+            flush=True,
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())