kis_bot/kis_trader/backtest/optuna_common.py

#!/usr/bin/env python3
"""
kis_trader/backtest/optuna_common.py — Optuna storage·DB 공통 (MariaDB 141)
=========================================================================
TradeDB(database.py) 와 동일 호스트·계정, 전용 DB kis_optuna 에 study 저장.
Win11·VM 양쪽에서 같은 storage 로 trial 공유·재개 가능.
"""
from __future__ import annotations

import logging
import os
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import quote_plus

from kis_trader.utils.env import get_env_float, get_env_from_db, get_env_int

logger = logging.getLogger("optuna_common")

# Optuna 전용 MariaDB (매매 DB kis_quant_db 와 분리)
DEFAULT_OPTUNA_DB_NAME = "kis_optuna"
OPTUNA_STRATEGIES = ("tail", "momentum", "us_momentum", "breakout", "scalp", "dart")

# 탐색(TPE 학습): 게이트 OFF(0) — PnL 차이를 샘플러가 보도록.
# 리포트/apply 후보: 아래 REPORT_* 로 사후 필터.
OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT = 0.0
OPTUNA_SEARCH_MIN_PF_DEFAULT = 0.0
OPTUNA_SEARCH_MIN_TRADES_DEFAULT = 1
OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT = 40.0
OPTUNA_REPORT_MIN_PF_DEFAULT = 1.0

# 일별 손익 안정성 티어 (results_stable) — 학습1위/gated 와 별도 후보
# 손실일·최악일·일PnL 분산으로 “합산만 큰” 후보를 걸러낸다.
OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT = 1
OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT = -30000.0
OPTUNA_STABLE_LAMBDA_DEFAULT = 1.0
OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT = 2


def optuna_search_gate_defaults() -> Tuple[float, float, int]:
    """탐색 중 objective 게이트 (기본 0/0/1). CLI 미지정 시 사용."""
    return (
        float(get_env_float("PARAM_SEARCH_OPTUNA_MIN_WIN_RATE", OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT)),
        float(get_env_float("PARAM_SEARCH_OPTUNA_MIN_PF", OPTUNA_SEARCH_MIN_PF_DEFAULT)),
        int(get_env_int("PARAM_SEARCH_OPTUNA_MIN_TRADES", OPTUNA_SEARCH_MIN_TRADES_DEFAULT)),
    )


def optuna_report_gate_defaults() -> Tuple[float, float, int]:
    """결과 후보·apply 사후 필터 (기본 승률40·PF1.0·min_trades=탐색과 동일)."""
    _sw, _sp, min_tr = optuna_search_gate_defaults()
    return (
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_REPORT_MIN_WIN_RATE", OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT,
        )),
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_REPORT_MIN_PF", OPTUNA_REPORT_MIN_PF_DEFAULT,
        )),
        int(get_env_int("PARAM_SEARCH_OPTUNA_REPORT_MIN_TRADES", max(1, min_tr))),
    )


def _sort_optuna_rows(rows: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]:
    sb = (sort_by or "pnl").strip().lower()
    out = list(rows)

    def _f(r: Dict[str, Any], k: str) -> float:
        try:
            return float(r.get(k) or 0)
        except (TypeError, ValueError):
            return 0.0

    if sb == "score":
        out.sort(key=lambda r: (-_f(r, "score"), -_f(r, "total_pnl"), -_f(r, "win_rate")))
    elif sb == "win_rate":
        out.sort(key=lambda r: (-_f(r, "win_rate"), -_f(r, "total_pnl")))
    elif sb in ("stability", "stable"):
        # 일평균 − λ·표준편차(stability_score) 우선 · 최악일 · 합산 PnL
        out.sort(
            key=lambda r: (
                -_f(r, "stability_score"),
                -_f(r, "worst_day_pnl"),
                -_f(r, "total_pnl"),
                -_f(r, "win_rate"),
            ),
        )
    else:
        out.sort(key=lambda r: (-_f(r, "total_pnl"), -_f(r, "win_rate")))
    return out


def trade_exit_day_key(trade: Dict[str, Any]) -> str:
    """청산 시각 → YYYY-MM-DD (없으면 빈 문자열).

    꼬리 백테는 exit_time, 스캘핑·모멘텀·돌파 포트폴리오 백테는 sell_time 을 씀.
    sell_time 누락 시 daily_pnl/results_stable 이 전부 비게 됨.
    """
    raw = (
        trade.get("exit_time")
        or trade.get("sell_date")
        or trade.get("sell_time")  # scalp/momentum/breakout 포트폴리오
        or trade.get("exit_ts")
        or trade.get("exit_at")
        or ""
    )
    s = str(raw).strip()
    if not s:
        return ""
    digits = "".join(ch for ch in s if ch.isdigit())
    if len(digits) >= 8:
        return f"{digits[0:4]}-{digits[4:6]}-{digits[6:8]}"
    if len(s) >= 10 and s[4] == "-" and s[7] == "-":
        return s[:10]
    return ""


def compute_daily_stability_metrics(
    trades: List[Dict[str, Any]],
    *,
    stability_lambda: Optional[float] = None,
) -> Dict[str, Any]:
    """
    거래 리스트 → 일별 PnL·안정성 점수.

    stability_score = mean(일PnL) − λ × std(일PnL)
    (λ 기본 OPTUNA_STABLE_LAMBDA / get_env)
    """
    from statistics import mean, pstdev

    if stability_lambda is None:
        _, _, lam, _ = optuna_stable_gate_defaults()
        stability_lambda = lam
    try:
        lam = float(stability_lambda)
    except (TypeError, ValueError):
        lam = float(OPTUNA_STABLE_LAMBDA_DEFAULT)

    by_day: Dict[str, float] = {}
    for t in trades or []:
        day = trade_exit_day_key(t if isinstance(t, dict) else {})
        if not day:
            continue
        try:
            pnl = float((t or {}).get("pnl") or (t or {}).get("realized_pnl") or 0)
        except (TypeError, ValueError):
            pnl = 0.0
        by_day[day] = by_day.get(day, 0.0) + pnl

    days_sorted = sorted(by_day.keys())
    vals = [float(by_day[d]) for d in days_sorted]
    n_days = len(vals)
    if n_days <= 0:
        return {
            "daily_pnl": {},
            "n_active_days": 0,
            "n_losing_days": 0,
            "worst_day_pnl": 0.0,
            "best_day_pnl": 0.0,
            "daily_pnl_mean": 0.0,
            "daily_pnl_std": 0.0,
            "stability_score": 0.0,
            "stability_lambda": lam,
        }

    n_lose = sum(1 for v in vals if v < 0)
    worst = min(vals)
    best = max(vals)
    avg = float(mean(vals))
    std = float(pstdev(vals)) if n_days >= 2 else 0.0
    score = avg - lam * std
    return {
        "daily_pnl": {d: round(by_day[d], 2) for d in days_sorted},
        "n_active_days": n_days,
        "n_losing_days": int(n_lose),
        "worst_day_pnl": round(worst, 2),
        "best_day_pnl": round(best, 2),
        "daily_pnl_mean": round(avg, 2),
        "daily_pnl_std": round(std, 2),
        "stability_score": round(score, 4),
        "stability_lambda": lam,
    }


def attach_daily_stability(
    result: Dict[str, Any],
    trades: List[Dict[str, Any]],
) -> Dict[str, Any]:
    """evaluate_* 반환 dict 에 일별 안정성 필드를 붙인다."""
    if not isinstance(result, dict):
        return result
    result.update(compute_daily_stability_metrics(trades or []))
    return result


def attach_optional_backtest_trades(
    result: Dict[str, Any],
    trades: List[Dict[str, Any]],
    include_trades: bool = False,
) -> Dict[str, Any]:
    """Optuna 후처리용. include_trades=False 면 기존과 동일(JSON/trial attrs 비대화 방지)."""
    if include_trades and isinstance(result, dict):
        result["_trades"] = list(trades or [])
    return result


def optuna_stable_gate_defaults() -> Tuple[int, float, float, int]:
    """(max_losing_days, min_worst_day_pnl, lambda, min_active_days)."""
    return (
        int(get_env_int(
            "PARAM_SEARCH_OPTUNA_STABLE_MAX_LOSING_DAYS",
            OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT,
        )),
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_STABLE_MIN_WORST_DAY_PNL",
            OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT,
        )),
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_STABLE_LAMBDA",
            OPTUNA_STABLE_LAMBDA_DEFAULT,
        )),
        int(get_env_int(
            "PARAM_SEARCH_OPTUNA_STABLE_MIN_ACTIVE_DAYS",
            OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT,
        )),
    )


def row_passes_report_gates(
    row: Dict[str, Any],
    *,
    min_win_rate: float,
    min_pf: float,
    min_trades: int,
) -> bool:
    try:
        wr = float(row.get("win_rate") or 0)
        pf = float(row.get("pf") or 0)
        nt = int(row.get("total_trades") or 0)
    except (TypeError, ValueError):
        return False
    if nt < int(min_trades):
        return False
    if wr < float(min_win_rate):
        return False
    if pf < float(min_pf):
        return False
    return True


def row_passes_stable_gates(row: Dict[str, Any]) -> bool:
    """
    일별 안정성 사후 게이트.
    daily_pnl / n_active_days 가 없으면(구 JSON) 통과 불가 → results_stable 빈 목록.
    """
    if row.get("daily_pnl") is None and row.get("n_active_days") is None:
        return False
    max_lose, min_worst, _lam, min_days = optuna_stable_gate_defaults()
    try:
        n_days = int(row.get("n_active_days") or 0)
        n_lose = int(row.get("n_losing_days") or 0)
        worst = float(row.get("worst_day_pnl") or 0)
    except (TypeError, ValueError):
        return False
    if n_days < int(min_days):
        return False
    if n_lose > int(max_lose):
        return False
    if worst < float(min_worst):
        return False
    return True


def set_optuna_trial_stability_attrs(trial: Any, result: Dict[str, Any]) -> None:
    """Optuna trial.user_attrs 에 일별 안정성 스냅샷 저장."""
    import json as _json

    if not result:
        return
    try:
        trial.set_user_attr("n_active_days", int(result.get("n_active_days") or 0))
        trial.set_user_attr("n_losing_days", int(result.get("n_losing_days") or 0))
        trial.set_user_attr("worst_day_pnl", float(result.get("worst_day_pnl") or 0))
        trial.set_user_attr("best_day_pnl", float(result.get("best_day_pnl") or 0))
        trial.set_user_attr("daily_pnl_mean", float(result.get("daily_pnl_mean") or 0))
        trial.set_user_attr("daily_pnl_std", float(result.get("daily_pnl_std") or 0))
        trial.set_user_attr("stability_score", float(result.get("stability_score") or 0))
        trial.set_user_attr(
            "daily_pnl_json",
            _json.dumps(result.get("daily_pnl") or {}, ensure_ascii=False),
        )
    except Exception:
        pass


def stability_fields_from_trial_attrs(trial: Any) -> Dict[str, Any]:
    """trial.user_attrs → 결과 row 안정성 필드."""
    import json as _json

    raw = trial.user_attrs.get("daily_pnl_json") or "{}"
    try:
        daily = _json.loads(raw) if isinstance(raw, str) else (raw or {})
    except Exception:
        daily = {}
    if trial.user_attrs.get("n_active_days") is None and not daily:
        return {}
    return {
        "daily_pnl": daily if isinstance(daily, dict) else {},
        "n_active_days": int(trial.user_attrs.get("n_active_days") or 0),
        "n_losing_days": int(trial.user_attrs.get("n_losing_days") or 0),
        "worst_day_pnl": float(trial.user_attrs.get("worst_day_pnl") or 0),
        "best_day_pnl": float(trial.user_attrs.get("best_day_pnl") or 0),
        "daily_pnl_mean": float(trial.user_attrs.get("daily_pnl_mean") or 0),
        "daily_pnl_std": float(trial.user_attrs.get("daily_pnl_std") or 0),
        "stability_score": float(trial.user_attrs.get("stability_score") or 0),
    }


def build_results_stable_tier(
    rows: List[Dict[str, Any]],
    *,
    top_n: int = 10,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    """
    안정 Top — 사후합격(gated)·플러스 PnL 과 독립 (마이너스도 상대 순위).

    1) 일별 안정 게이트 통과분 → 안정점수순
    2) 0건이면 학습풀 전체를 안정점수순 TopN (게이트는 참고·폴백 표시)
       → 3일 전패장에서 max_losing_days=1 이면 게이트 0이어도 표가 비지 않음
    """
    max_lose, min_worst, lam, min_days = optuna_stable_gate_defaults()
    n = max(1, int(top_n or 10))
    all_rows = [r for r in (rows or []) if isinstance(r, dict)]
    pool = [r for r in all_rows if row_passes_stable_gates(r)]
    fallback = False
    if pool:
        stable = _sort_optuna_rows(pool, "stability")[:n]
    else:
        fallback = True
        with_stab = [
            r for r in all_rows
            if r.get("stability_score") is not None
            or r.get("n_active_days") is not None
            or r.get("daily_pnl") is not None
        ]
        src = with_stab if with_stab else all_rows
        stable = _sort_optuna_rows(src, "stability")[:n]
    meta = {
        "max_losing_days": max_lose,
        "min_worst_day_pnl": min_worst,
        "stability_lambda": lam,
        "min_active_days": min_days,
        "score_note": "stability_score = mean(일PnL) − λ × std(일PnL)",
        "fallback_rank_only": bool(fallback),
        "n_gate_pass": len(pool),
        "fallback_note": (
            "안정 게이트 0건 → 학습풀 안정점수순 TopN (마이너스 PnL 포함 · 상대비교)"
            if fallback else ""
        ),
    }
    return stable, meta


def resolve_results_stable(
    data: Optional[Dict[str, Any]],
    *,
    top_n: Optional[int] = None,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    """JSON results_stable 우선 · 비면 학습풀에서 즉시 재구성 (구잡·gated=0 공용)."""
    data = data or {}
    try:
        n = int(top_n) if top_n is not None else 10
    except (TypeError, ValueError):
        n = 10
    n = max(1, n)
    stored = [r for r in list(data.get("results_stable") or []) if isinstance(r, dict)]
    gates = dict(data.get("stable_gates") or {})
    if stored:
        return stored[:n], gates
    allr = list(data.get("results_all") or data.get("results") or [])
    stable, meta = build_results_stable_tier(allr, top_n=n)
    gates.update(meta)
    return stable, gates


def build_optuna_result_tiers(
    rows: List[Dict[str, Any]],
    *,
    sort_by: str,
    top_n: int = 5000,
) -> Dict[str, Any]:
    """
    탐색 전체 vs 리포트/apply 후보 분리.

    - results_all: 완료·게이트통과(탐색게이트) trial 전부 정렬
    - results: 하위호환 — 플러스 PnL 우선(없으면 all)
    - results_gated: 승률·PF 사후 필터 (apply 후보, PnL>0)
    - results_stable: 학습풀 일별 안정성 (gated·플러스 독립 · 게이트0이면 점수순 폴백)
    """
    rep_wr, rep_pf, rep_tr = optuna_report_gate_defaults()
    all_sorted = _sort_optuna_rows(rows, sort_by)
    profitable = [r for r in all_sorted if float(r.get("total_pnl") or 0) > 0]
    learning = profitable if profitable else all_sorted
    gated = [
        r for r in all_sorted
        if row_passes_report_gates(
            r, min_win_rate=rep_wr, min_pf=rep_pf, min_trades=rep_tr,
        )
        and float(r.get("total_pnl") or 0) > 0
    ]
    # 안정 TopN 표용 — 전체 풀에서 상위 (gated 잘림·플러스와 무관)
    try:
        stable_ui_n = max(1, int(get_env_int("OPTUNA_POST_TOP_N", 10)))
    except Exception:
        stable_ui_n = 10
    stable, stable_meta = build_results_stable_tier(all_sorted, top_n=stable_ui_n)
    return {
        "results_all": all_sorted[:top_n],
        "results": learning[:top_n],
        "results_gated": gated[:top_n],
        "results_stable": stable,
        "report_gates": {
            "min_win_rate": rep_wr,
            "min_pf": rep_pf,
            "min_trades": rep_tr,
        },
        "stable_gates": stable_meta,
        "search_gates_note": (
            "탐색 min_win_rate/min_pf 기본 0 — TPE가 PnL 차이를 학습. "
            "적용·운영 후보는 results_gated(report_gates). "
            "들쭉날쭉 완화·상대비교는 results_stable (gated/플러스 독립 · 게이트0이면 점수순 폴백)."
        ),
        "n_results_all": len(all_sorted),
        "n_results_learning": len(learning),
        "n_results_gated": len(gated),
        "n_results_stable": len(stable),
    }


def _optuna_overfit_sample_days(data: Dict[str, Any]) -> int:
    try:
        days = int(data.get("backtest_days") or 0)
    except (TypeError, ValueError):
        days = 0
    if days > 0:
        return days
    start = str(data.get("start") or "")
    end = str(data.get("end") or "")
    try:
        from datetime import datetime as _dt

        return max(
            1,
            (_dt.strptime(end, "%Y-%m-%d") - _dt.strptime(start, "%Y-%m-%d")).days + 1,
        )
    except Exception:
        return 1


def overfit_risk_pct_for_row(
    data: Dict[str, Any],
    row: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
    """
    후보 한 줄의 과적합 가능도% (0~100, 높을수록 위험 · 만점=100).

    스터디 공통(표본 장일) + 이 trial의 거래수·승률·PF 이상치.
    교차검증이 아님. DB apply 게이트와 별개.
    """
    def _f(x: Any, default: float = 0.0) -> float:
        try:
            return float(x)
        except (TypeError, ValueError):
            return default

    def _i(x: Any, default: int = 0) -> int:
        try:
            return int(x)
        except (TypeError, ValueError):
            return default

    days = _optuna_overfit_sample_days(data or {})
    risk = 0.0
    if days <= 1:
        risk += 40.0
    elif days == 2:
        risk += 28.0
    elif days <= 4:
        risk += 16.0

    if not row:
        risk = max(0.0, min(100.0, round(risk + 25.0, 1)))
        return {
            "overfit_risk_pct": risk,
            "verdict": "비권장",
            "verdict_ui": "위험 · 비권장",
        }

    nt = _i(row.get("total_trades"))
    wr = _f(row.get("win_rate"))
    pf = _f(row.get("pf"))
    pnl = _f(row.get("total_pnl"))

    if nt <= 1:
        risk += 25.0
    elif nt <= 3:
        risk += 18.0
    elif nt <= 9:
        risk += 10.0

    if wr >= 90.0 and nt < 10:
        risk += 15.0
    elif wr >= 80.0 and nt < 5:
        risk += 10.0
    if pf >= 50.0 and nt < 10:
        risk += 10.0
    elif pf >= 10.0 and nt < 5:
        risk += 6.0

    gated = list((data or {}).get("results_gated") or [])
    learn = list((data or {}).get("results") or (data or {}).get("results_all") or [])
    pool = gated if gated else learn
    if pool and nt > 0:
        best_pnl = round(pnl, 0)
        same = [r for r in pool if abs(_f(r.get("total_pnl")) - best_pnl) < 1.0]
        share = len(same) / max(1, len(pool))
        if share >= 0.4 and len(same) >= 5:
            risk += 12.0
        elif share >= 0.25 and len(same) >= 3:
            risk += 6.0

    risk = max(0.0, min(100.0, round(risk, 1)))
    if risk >= 70.0:
        verdict, verd_ui = "비권장", "위험 · 비권장"
    elif risk >= 40.0:
        verdict, verd_ui = "주의", "주의"
    else:
        verdict, verd_ui = "상대적으로낮음", "상대적으로 낮음"
    return {
        "overfit_risk_pct": risk,
        "verdict": verdict,
        "verdict_ui": verd_ui,
    }


def build_optuna_overfit_diagnostics(data: Dict[str, Any]) -> Dict[str, Any]:
    """
    Optuna 결과 → 과적합 위험% · 적용 가능도% · 임계값(파라미터) 분포 표용 dict.

    - 통계적 교차검증이 아니라 **운영 휴리스틱**(표본 일수·거래수·승률/PF 이상치·평탄 고원).
    - 높을수록 과적합 위험. 적용 가능도 ≈ 100 − 위험 (하한 0).
    - 웹·브리핑·JSON 공통. DB apply 게이트는 바꾸지 않음(표시·판별용).
    """
    import statistics

    def _f(x: Any, default: float = 0.0) -> float:
        try:
            return float(x)
        except (TypeError, ValueError):
            return default

    def _i(x: Any, default: int = 0) -> int:
        try:
            return int(x)
        except (TypeError, ValueError):
            return default

    days = _i(data.get("backtest_days"), 0)
    if days <= 0:
        start = str(data.get("start") or "")
        end = str(data.get("end") or "")
        try:
            from datetime import datetime as _dt

            days = max(
                1,
                (_dt.strptime(end, "%Y-%m-%d") - _dt.strptime(start, "%Y-%m-%d")).days + 1,
            )
        except Exception:
            days = 1

    gated = list(data.get("results_gated") or [])
    learn = list(data.get("results") or data.get("results_all") or [])
    pool = gated if gated else learn
    top = pool[0] if pool else None
    n_gated = _i(data.get("n_results_gated"), len(gated))
    n_all = _i(data.get("n_results_all"), len(data.get("results_all") or learn))
    n_stable = _i(data.get("n_results_stable"), len(data.get("results_stable") or []))

    factors: List[Dict[str, Any]] = []
    risk = 0.0

    # 1) 표본 장일
    if days <= 1:
        pts, detail = 40.0, f"거래일≈{days}일 — 단일 장 과적합 위험 최대"
    elif days == 2:
        pts, detail = 28.0, f"거래일≈{days}일 — 이틀만으로는 추세 전환에 취약"
    elif days <= 4:
        pts, detail = 16.0, f"거래일≈{days}일 — 다일 재검증 권장(≥5일)"
    else:
        pts, detail = 0.0, f"거래일≈{days}일 — 표본 일수 상대적 양호"
    risk += pts
    factors.append({"id": "sample_days", "label": "표본 장일", "points": pts, "detail": detail})

    nt = _i(top.get("total_trades")) if top else 0
    wr = _f(top.get("win_rate")) if top else 0.0
    pf = _f(top.get("pf")) if top else 0.0
    pnl = _f(top.get("total_pnl")) if top else 0.0

    # 2) 거래 표본
    if not top:
        pts, detail = 25.0, "통과 후보 없음 — 적용 불가"
    elif nt <= 1:
        pts, detail = 25.0, f"상위 후보 거래 {nt}건 — 우연 승·과적합 가능"
    elif nt <= 3:
        pts, detail = 18.0, f"상위 후보 거래 {nt}건 — 표본 부족"
    elif nt <= 9:
        pts, detail = 10.0, f"상위 후보 거래 {nt}건 — 해석 시 주의"
    else:
        pts, detail = 0.0, f"상위 후보 거래 {nt}건 — 상대적 양호"
    risk += pts
    factors.append({"id": "trade_count", "label": "거래 표본", "points": pts, "detail": detail})

    # 3) 승률/PF 이상치
    pts = 0.0
    bits: List[str] = []
    if top and wr >= 90.0 and nt < 10:
        pts += 15.0
        bits.append(f"승률 {wr:.1f}% + 거래 {nt}건")
    elif top and wr >= 80.0 and nt < 5:
        pts += 10.0
        bits.append(f"승률 {wr:.1f}% + 거래 {nt}건")
    if top and pf >= 50.0 and nt < 10:
        pts += 10.0
        bits.append(f"PF {pf:.2f} (소수 거래 폭증)")
    elif top and pf >= 10.0 and nt < 5:
        pts += 6.0
        bits.append(f"PF {pf:.2f}")
    detail = " · ".join(bits) if bits else "이상치 없음"
    risk += pts
    factors.append({"id": "outlier_wr_pf", "label": "승률·PF 이상치", "points": pts, "detail": detail})

    # 4) gated 부재 / 거의 전원 통과
    pts = 0.0
    if n_gated <= 0 and n_all > 0:
        pts = 12.0
        detail = f"사후합격 0건 (학습 {n_all}) — DB 적용 비권장"
    elif n_all > 0 and n_gated / max(1, n_all) >= 0.85 and days <= 2:
        pts = 10.0
        detail = f"gated/all={n_gated}/{n_all} — 단일에 대부분 통과(필터 느슨·노이즈)"
    elif n_gated > 0:
        pts = 0.0
        detail = f"사후합격 {n_gated}건 · stable {n_stable}건"
    else:
        pts = 8.0
        detail = "학습·gated 모두 비어 있음"
    risk += pts
    factors.append({"id": "gate_coverage", "label": "게이트 커버", "points": pts, "detail": detail})

    # 5) PnL 고원(동일 best 반복)
    plateau_share = 0.0
    plateau_n = 0
    if pool and top:
        best_pnl = round(pnl, 0)
        same = [
            r for r in pool
            if abs(_f(r.get("total_pnl")) - best_pnl) < 1.0
        ]
        plateau_n = len(same)
        plateau_share = plateau_n / max(1, len(pool))
        if plateau_share >= 0.4 and plateau_n >= 5:
            pts = 12.0
            detail = (
                f"동일 PnL≈{best_pnl:,.0f}원이 후보 {plateau_n}/{len(pool)} "
                f"({plateau_share:.0%}) — 파라미터 민감도 낮음/고원"
            )
        elif plateau_share >= 0.25 and plateau_n >= 3:
            pts = 6.0
            detail = f"PnL 고원 {plateau_n}/{len(pool)} ({plateau_share:.0%})"
        else:
            pts = 0.0
            detail = f"고원 비율 {plateau_share:.0%} ({plateau_n}건)"
    else:
        pts, detail = 0.0, "고원 판정 스킵"
    risk += pts
    factors.append({"id": "pnl_plateau", "label": "PnL 고원", "points": pts, "detail": detail})

    risk = max(0.0, min(100.0, round(risk, 1)))
    apply_pct = max(0.0, min(100.0, round(100.0 - risk, 1)))
    if risk >= 70.0:
        verdict = "비권장"
        verdict_ko = "과적합·표본부족 위험 높음 — 실매 DB 즉시 적용 비권장"
    elif risk >= 40.0:
        verdict = "주의"
        verdict_ko = "적용 가능도 중간 — 다일(≥5일) 재검증·웹백테 후 소액만"
    else:
        verdict = "상대적으로낮음"
        verdict_ko = "휴리스틱상 위험 상대적 낮음 — 그래도 다일 확인 권장"

    # --- 임계값 분포 (gated 우선, 상위 min(30, len) 행) ---
    dist_rows = pool[: min(30, len(pool))]
    skip_keys = {
        "params", "apply_cfg", "merged_params", "daily_pnl", "optuna_trial_number",
        "total_trades", "win_rate", "total_pnl", "pf", "score", "stability_score",
        "n_losing_days", "n_active_days", "worst_day_pnl", "best_day_pnl",
        "daily_pnl_mean", "daily_pnl_std", "skip_hts_scan_dupes",
    }
    prefer = list(data.get("grid_keys") or [])
    # 꼬리·공통에서 자주 보는 축
    prefer_extra = [
        "min_drop_rate", "min_recovery_ratio", "tail_ratio_min", "tail_pct_min",
        "stop_atr_mult", "target_atr_mult", "atr_sl_min_pct", "atr_sl_max_pct",
        "atr_tp_min_pct", "atr_tp_max_pct", "rsi_threshold", "bar_chg_min_pct",
        "bar_chg_max_pct", "shoulder_min_high", "shoulder_cut_pct", "cooldown_min",
        "max_daily", "whipsaw_enabled", "ratchet_on", "sl_pct", "tp_pct",
    ]
    key_order = []
    for k in prefer + prefer_extra:
        if k not in key_order:
            key_order.append(k)

    # 실제 등장 키 수집
    value_maps: Dict[str, List[Any]] = {}
    for row in dist_rows:
        params = row.get("merged_params") or row.get("params") or {}
        if not isinstance(params, dict):
            continue
        for k, v in params.items():
            if k in skip_keys or str(k).startswith("_"):
                continue
            value_maps.setdefault(str(k), []).append(v)

    def _percentile(sorted_vals: List[float], p: float) -> float:
        if not sorted_vals:
            return 0.0
        if len(sorted_vals) == 1:
            return sorted_vals[0]
        idx = (len(sorted_vals) - 1) * p
        lo = int(idx)
        hi = min(lo + 1, len(sorted_vals) - 1)
        w = idx - lo
        return sorted_vals[lo] * (1.0 - w) + sorted_vals[hi] * w

    threshold_distribution: List[Dict[str, Any]] = []
    keys_out = [k for k in key_order if k in value_maps]
    # prefer 외 숫자 키 보충 (최대 18개 표시)
    for k in sorted(value_maps.keys()):
        if k not in keys_out:
            keys_out.append(k)
        if len(keys_out) >= 18:
            break

    for k in keys_out:
        vals = value_maps.get(k) or []
        if not vals:
            continue
        # bool / categorical
        as_num: List[float] = []
        for v in vals:
            if isinstance(v, bool):
                as_num.append(1.0 if v else 0.0)
            else:
                try:
                    as_num.append(float(v))
                except (TypeError, ValueError):
                    as_num = []
                    break
        # mode
        try:
            mode_v = statistics.mode(vals)
        except statistics.StatisticsError:
            mode_v = vals[0]
        mode_n = sum(1 for v in vals if v == mode_v)
        mode_share = mode_n / max(1, len(vals))
        row_d: Dict[str, Any] = {
            "param": k,
            "n": len(vals),
            "mode": mode_v,
            "mode_share": round(mode_share, 3),
        }
        if as_num:
            s = sorted(as_num)
            row_d["p25"] = round(_percentile(s, 0.25), 6)
            row_d["median"] = round(_percentile(s, 0.50), 6)
            row_d["p75"] = round(_percentile(s, 0.75), 6)
            row_d["min"] = round(s[0], 6)
            row_d["max"] = round(s[-1], 6)
        else:
            row_d["p25"] = None
            row_d["median"] = None
            row_d["p75"] = None
            row_d["min"] = None
            row_d["max"] = None
        threshold_distribution.append(row_d)

    pool_tag = "results_gated" if gated else "results(learning)"
    return {
        "overfit_risk_pct": risk,
        "apply_readiness_pct": apply_pct,
        "verdict": verdict,
        "verdict_ko": verdict_ko,
        "sample_days": days,
        "n_gated": n_gated,
        "n_all": n_all,
        "n_stable": n_stable,
        "top_trades": nt,
        "top_win_rate": wr,
        "top_pf": pf,
        "top_pnl": pnl,
        "plateau_share": round(plateau_share, 3),
        "plateau_n": plateau_n,
        "factors": factors,
        "threshold_distribution": threshold_distribution,
        "threshold_pool": pool_tag,
        "threshold_pool_n": len(dist_rows),
        "note": (
            "과적합%는 교차검증 점수가 아니라 표본·이상치·고원 휴리스틱입니다. "
            "적용 가능도%=100−과적합위험%. DB 적용 버튼 활성 조건(gated PnL>0)과는 별개입니다."
        ),
    }


def attach_optuna_overfit_diagnostics(data: Dict[str, Any]) -> Dict[str, Any]:
    """JSON dict 에 overfit_diagnostics 키를 채운다 (있으면 갱신)."""
    try:
        data["overfit_diagnostics"] = build_optuna_overfit_diagnostics(data)
    except Exception as exc:
        logger.warning("⚠️ overfit_diagnostics 생성 실패: %s", exc)
        data["overfit_diagnostics"] = {
            "overfit_risk_pct": None,
            "apply_readiness_pct": None,
            "verdict": "error",
            "verdict_ko": f"진단 실패: {exc}",
            "factors": [],
            "threshold_distribution": [],
            "note": str(exc),
        }
    return data


def pick_gated_apply_trial(
    study: Any,
    *,
    sort_by: str = "pnl",
    fail_objective: float = -1e18,
) -> Optional[Any]:
    """
    --apply-best 용: study.best(탐색 objective)가 아니라
    report_gates 통과 trial 중 정렬 1위.
    """
    import optuna  # noqa: WPS433 — 호출 시에만

    rep_wr, rep_pf, rep_tr = optuna_report_gate_defaults()
    cand: List[Tuple[Dict[str, Any], Any]] = []
    for trial in study.trials:
        if trial.state != optuna.trial.TrialState.COMPLETE:
            continue
        if not trial.user_attrs.get("gates_ok"):
            continue
        try:
            val = float(trial.value) if trial.value is not None else fail_objective
        except (TypeError, ValueError):
            val = fail_objective
        if val <= fail_objective + 1:
            continue
        row = {
            "win_rate": float(trial.user_attrs.get("win_rate") or 0),
            "pf": float(trial.user_attrs.get("pf") or 0),
            "total_trades": int(trial.user_attrs.get("total_trades") or 0),
            "total_pnl": float(trial.user_attrs.get("total_pnl") or 0),
            "score": float(trial.user_attrs.get("score") or 0),
            "_trial_number": int(trial.number),
        }
        if not row_passes_report_gates(
            row, min_win_rate=rep_wr, min_pf=rep_pf, min_trades=rep_tr,
        ):
            continue
        if float(row["total_pnl"]) <= 0:
            continue
        cand.append((row, trial))
    if not cand:
        return None
    ranked = _sort_optuna_rows([r for r, _ in cand], sort_by)
    top_n = int(ranked[0].get("_trial_number") or -1)
    for r, t in cand:
        if int(r.get("_trial_number") or -2) == top_n:
            return t
    return cand[0][1]


def ensure_optuna_gate_env_defaults(db: Any = None) -> None:
    """신규 Optuna 게이트 키가 DB에 없으면 env_config_ext 에만 UPSERT (전체 스냅샷 X)."""
    defaults = {
        "PARAM_SEARCH_OPTUNA_MIN_WIN_RATE": str(OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT),
        "PARAM_SEARCH_OPTUNA_MIN_PF": str(OPTUNA_SEARCH_MIN_PF_DEFAULT),
        "PARAM_SEARCH_OPTUNA_MIN_TRADES": str(OPTUNA_SEARCH_MIN_TRADES_DEFAULT),
        "PARAM_SEARCH_OPTUNA_REPORT_MIN_WIN_RATE": str(OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT),
        "PARAM_SEARCH_OPTUNA_REPORT_MIN_PF": str(OPTUNA_REPORT_MIN_PF_DEFAULT),
        "PARAM_SEARCH_OPTUNA_BRIEFING_AI": "1",
        # 일별 안정성 티어 (results_stable)
        "PARAM_SEARCH_OPTUNA_STABLE_MAX_LOSING_DAYS": str(OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT),
        "PARAM_SEARCH_OPTUNA_STABLE_MIN_WORST_DAY_PNL": str(OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT),
        "PARAM_SEARCH_OPTUNA_STABLE_LAMBDA": str(OPTUNA_STABLE_LAMBDA_DEFAULT),
        "PARAM_SEARCH_OPTUNA_STABLE_MIN_ACTIVE_DAYS": str(OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT),
        # Optuna apply 시 다단트레일 추천 → 전략별 *_DAILY_PROFIT_* (탐색 축 아님)
        "OPTUNA_DAILY_TRAIL_APPLY_ON_BEST": "true",
        "OPTUNA_DAILY_TRAIL_ARM_FRAC": "0.60",
        "OPTUNA_DAILY_TRAIL_BEST_FRAC": "0.70",
        "OPTUNA_DAILY_TRAIL_ARM_STEP": "5000",
        "OPTUNA_DAILY_TRAIL_MIN_ARM": "10000",
        "OPTUNA_DAILY_TRAIL_TIER_DROPS": "40,30,20",
        "OPTUNA_POST_TOP_N": "10",
        "OPTUNA_POST_INCLUDE_MODE": "true",
        "OPTUNA_POST_INCLUDE_LIVE": "true",
        "OPTUNA_POST_INCLUDE_STABLE": "true",
        "OPTUNA_POST_RUN_OB_WHIPSAW": "true",
        "OPTUNA_OB_RECOMMEND_TRIALS": "500",
        "OPTUNA_OB_AXIS_TRIALS": "500",
        "OPTUNA_OB_COMBO_TRIALS_SINGLE": "150",
        "OPTUNA_OB_COMBO_TRIALS_DOUBLE": "200",
        "OPTUNA_OB_COMBO_TRIALS_TRIPLE": "250",
        "OPTUNA_OB_ENTRY_SPREAD_MIN": "0.1",
        "OPTUNA_OB_ENTRY_SPREAD_MAX": "8.0",
        "OPTUNA_OB_ENTRY_RATIO_MIN": "0.05",
        "OPTUNA_OB_ENTRY_RATIO_MAX": "1.5",
        "OPTUNA_OB_ENTRY_ASK_MULT_MIN": "1.0",
        "OPTUNA_OB_ENTRY_ASK_MULT_MAX": "80.0",
        "OPTUNA_OB_LOOKBACK_MIN": "30",
        "OPTUNA_OB_EXIT_HOLD_MIN": "1",
        "OPTUNA_OB_EXIT_HOLD_MAX": "5",
        "OPTUNA_OB_EXIT_RATIO_MIN": "0.2",
        "OPTUNA_OB_EXIT_RATIO_MAX": "0.8",
        "OPTUNA_OB_EXIT_PROFIT_MIN": "0.003",
        "OPTUNA_OB_EXIT_PROFIT_MAX": "0.02",
        "OPTUNA_OB_EXIT_MA_MIN": "3",
        "OPTUNA_OB_EXIT_MA_MAX": "10",
        "OPTUNA_OB_STOP_HOLD_MIN": "1",
        "OPTUNA_OB_STOP_HOLD_MAX": "5",
        "OPTUNA_OB_STOP_RATIO_MIN": "0.2",
        "OPTUNA_OB_STOP_RATIO_MAX": "0.8",
        "OPTUNA_OB_STOP_LOSS_MIN": "0.001",
        "OPTUNA_OB_STOP_LOSS_MAX": "0.02",
        "OPTUNA_OB_STOP_MA_MIN": "3",
        "OPTUNA_OB_STOP_MA_MAX": "10",
        "OPTUNA_WHIPSAW_RECOMMEND_TRIALS": "500",
        "OPTUNA_OB_HORIZON_MIN": "6",
        "OPTUNA_WHIPSAW_LOOKBACK_DAYS": "7",
        "OPTUNA_WHIPSAW_TICK_LOOKBACK_SEC": "180",
    }
    try:
        from datetime import datetime

        from database import TradeDB
    except ImportError:
        return
    owned = False
    if db is None:
        db = TradeDB()
        owned = True
    try:
        snap = db.get_merged_env_snapshot() or {}
        patch = {}
        for k, v in defaults.items():
            cur = snap.get(k)
            if cur is None or str(cur).strip() == "":
                patch[k] = v
        if not patch:
            return
        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        n = db._persist_env_config_overflow(patch, now)
        try:
            from kis_trader.utils.env import invalidate_merged_env_cache
            invalidate_merged_env_cache()
        except Exception:
            pass
        logger.info(
            "📌 Optuna 게이트 기본값 DB(ext) 반영 %d키: %s",
            n, sorted(patch.keys()),
        )
    except Exception as exc:
        logger.warning("⚠️ Optuna 게이트 기본값 DB 반영 실패: %s", exc)
    finally:
        if owned:
            try:
                db.conn.close()
            except Exception:
                pass


def mariadb_creds() -> dict:
    """TradeDB(database.py) 와 동일 우선순위 — env > 기본 141."""
    return {
        "host": os.environ.get("DB_HOST", "192.168.0.141"),
        "port": int(os.environ.get("DB_PORT", "3306")),
        "user": os.environ.get("DB_USER", "jae"),
        "password": os.environ.get("DB_PASS", "1234"),
    }


def resolve_optuna_db_name() -> str:
    """
    Optuna storage DB — 기본 kis_optuna (매매 kis_quant_db 와 분리).
    env OPTUNA_DB_NAME 로 오버라이드 가능.
    """
    raw = get_env_from_db("OPTUNA_DB_NAME", "")
    if raw and str(raw).strip() not in ("", "None"):
        return str(raw).strip()
    env = os.environ.get("OPTUNA_DB_NAME", "")
    if env and str(env).strip():
        return str(env).strip()
    return DEFAULT_OPTUNA_DB_NAME


def build_mariadb_storage_url(db_name: Optional[str] = None) -> str:
    """mysql+pymysql://…@141/optuna 형식 storage URL."""
    creds = mariadb_creds()
    name = (db_name or resolve_optuna_db_name()).strip()
    user = quote_plus(creds["user"])
    passwd = quote_plus(creds["password"])
    return (
        f"mysql+pymysql://{user}:{passwd}@{creds['host']}:{creds['port']}/{name}"
        f"?charset=utf8mb4"
    )


def ensure_optuna_database(db_name: Optional[str] = None) -> str:
    """
    MariaDB 141 — kis_optuna 존재 확인 (없으면 CREATE 시도).
    """
    name = (db_name or resolve_optuna_db_name()).strip()
    creds = mariadb_creds()

    try:
        import pymysql
    except ImportError as exc:
        raise ImportError(
            "Optuna MariaDB storage 는 pymysql 필요: pip install PyMySQL"
        ) from exc

    # DB 존재 여부만 확인 (이미 있으면 CREATE 생략)
    conn = pymysql.connect(
        host=creds["host"],
        port=creds["port"],
        user=creds["user"],
        password=creds["password"],
        charset="utf8mb4",
        autocommit=True,
        connect_timeout=10,
    )
    try:
        with conn.cursor() as cur:
            cur.execute("SHOW DATABASES LIKE %s", (name,))
            exists = cur.fetchone() is not None
            if not exists:
                cur.execute(
                    f"CREATE DATABASE IF NOT EXISTS `{name}` "
                    "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
                )
                logger.info(
                    "📦 Optuna DB 생성: %s@%s:%s/%s",
                    creds["user"], creds["host"], creds["port"], name,
                )
            else:
                logger.debug(
                    "📦 Optuna storage DB: %s@%s:%s/%s",
                    creds["user"], creds["host"], creds["port"], name,
                )
    except Exception as exc:
        logger.error("❌ Optuna DB '%s' 접속/확인 실패: %s", name, exc)
        raise
    finally:
        conn.close()
    return name


def resolve_optuna_storage_url(cli_override: Optional[str] = None) -> str:
    """
    Storage URL 우선순위:
      1) CLI --storage
      2) OPTUNA_STORAGE_URL (DB/env)
      3) MariaDB 141 / kis_optuna (TradeDB 동일 계정)
    """
    if cli_override and str(cli_override).strip():
        return str(cli_override).strip()
    from_db = get_env_from_db("OPTUNA_STORAGE_URL", "")
    if from_db and str(from_db).strip() not in ("", "None"):
        return str(from_db).strip()
    db_name = ensure_optuna_database()
    return build_mariadb_storage_url(db_name)


def resolve_study_name(
    *,
    strategy: str,
    mode: str,
    start: str,
    end: str,
    cli_override: Optional[str] = None,
    extra: Optional[str] = None,
) -> str:
    """Study 이름 — 전략·기간·모드 포함. extra=꼬리 진입모드 등(스터디 분리)."""
    if cli_override and str(cli_override).strip():
        return str(cli_override).strip()
    env_key = f"OPTUNA_{strategy.upper()}_STUDY_NAME"
    from_db = get_env_from_db(env_key, "")
    if from_db and str(from_db).strip() not in ("", "None"):
        return str(from_db).strip()
    legacy = get_env_from_db("OPTUNA_TAIL_STUDY_NAME", "")
    if strategy == "tail" and legacy and str(legacy).strip() not in ("", "None"):
        return str(legacy).strip()
    extra_s = str(extra or "").strip().lower()
    extra_s = f"_{extra_s}" if extra_s else ""
    return f"{strategy}_{mode}{extra_s}_{start}_{end}"


def optuna_run_lock_name(strategy: str) -> str:
    return f"{strategy}_param_search_optuna"


def release_shared_tick_store(ctx: Any, *, log: Optional[logging.Logger] = None) -> None:
    """
    Optuna ctx.shared_tick_store 해제.

    주의: ticks_by_code 가 공유메모리 뷰인 경우, unlink 이후 접근하면
    SIGBUS/강제종료(트레이스백 없음) 난다. 최빈(mode_combo) 실측·JSON 저장이
    끝난 뒤에만 호출할 것. optimize() 직후 즉시 unlink 금지.
    """
    lg = log or logger
    store = getattr(ctx, "shared_tick_store", None)
    if store is None:
        return
    try:
        store.unlink()
    except Exception as exc:
        lg.warning("⚠️ shared_tick_store unlink 실패: %s", exc)
    try:
        ctx.shared_tick_store = None
    except Exception:
        pass



def announce_optuna_json_path(
    out_path: str,
    *,
    strategy: str = "",
    mode: str = "",
    note: str = "",
    log: Optional[logging.Logger] = None,
) -> str:
    """
    결과 JSON 절대경로를 터미널·로그에 눈에 띄게 고지.
    또한 logs/optuna_<strategy>_<mode>_latest.jsonpath 에 기록 (없으면 strategy만).
    note 에 '최종' 이 포함되면 이전장/앞장 브리핑(.briefing.md) 생성.
    """
    abs_path = os.path.abspath(str(out_path or "").strip())
    lg = log or logger
    tag = note.strip() or "결과 JSON"
    line = f"📁 [{tag}] {abs_path}"
    # logger + print 이중 — nohup 로그·터미널 모두에서 바로 보이게
    lg.info("%s", line)
    print(line, flush=True)
    print(f"OPTUNA_RESULT_JSON={abs_path}", flush=True)

    try:
        root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
        logs_dir = os.path.join(root, "logs")
        os.makedirs(logs_dir, exist_ok=True)
        s = (strategy or "optuna").strip().lower() or "optuna"
        m = (mode or "run").strip().lower() or "run"
        for name in (
            f"optuna_{s}_{m}_latest.jsonpath",
            f"optuna_{s}_latest.jsonpath",
            "optuna_latest.jsonpath",
        ):
            with open(os.path.join(logs_dir, name), "w", encoding="utf-8") as f:
                f.write(abs_path + "\n")
    except OSError as exc:
        lg.warning("⚠️ jsonpath 사이드카 기록 실패: %s", exc)

    # 최종 JSON: 과적합·임계값 분포 진단 부착 후 브리핑
    note_l = (note or "").strip()
    if "최종" in note_l and abs_path and os.path.isfile(abs_path):
        try:
            import json as _json

            with open(abs_path, "r", encoding="utf-8") as f:
                _data = _json.load(f)
            attach_optuna_overfit_diagnostics(_data)
            with open(abs_path, "w", encoding="utf-8") as f:
                _json.dump(_data, f, indent=2, ensure_ascii=False)
            diag = _data.get("overfit_diagnostics") or {}
            lg.info(
                "📊 과적합위험 %s%% · 적용가능도 %s%% · 판정=%s",
                diag.get("overfit_risk_pct"),
                diag.get("apply_readiness_pct"),
                diag.get("verdict"),
            )
            print(
                f"OPTUNA_OVERFIT_RISK_PCT={diag.get('overfit_risk_pct')} "
                f"APPLY_READINESS_PCT={diag.get('apply_readiness_pct')} "
                f"VERDICT={diag.get('verdict')}",
                flush=True,
            )
        except Exception as exc:
            lg.warning("⚠️ overfit_diagnostics JSON 부착 실패: %s", exc)
        try:
            from kis_trader.backtest.optuna_briefing import write_briefing_for_json
            write_briefing_for_json(abs_path, log=lg)
        except Exception as exc:
            lg.warning("⚠️ Optuna 브리핑 실패: %s", exc)
    return abs_path