kis_bot/kis_trader/backtest/optuna_common.py

#!/usr/bin/env python3
"""
kis_trader/backtest/optuna_common.py — Optuna storage·DB 공통 (MariaDB 141)
=========================================================================
TradeDB(database.py) 와 동일 호스트·계정, 전용 DB kis_optuna 에 study 저장.
Win11·VM 양쪽에서 같은 storage 로 trial 공유·재개 가능.
"""
from __future__ import annotations

import logging
import os
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import quote_plus

from kis_trader.utils.env import get_env_float, get_env_from_db, get_env_int

logger = logging.getLogger("optuna_common")

# Optuna 전용 MariaDB (매매 DB kis_quant_db 와 분리)
DEFAULT_OPTUNA_DB_NAME = "kis_optuna"
OPTUNA_STRATEGIES = ("tail", "momentum", "us_momentum", "breakout", "scalp", "dart")

# 탐색(TPE 학습): 게이트 OFF(0) — PnL 차이를 샘플러가 보도록.
# 리포트/apply 후보: 아래 REPORT_* 로 사후 필터.
OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT = 0.0
OPTUNA_SEARCH_MIN_PF_DEFAULT = 0.0
OPTUNA_SEARCH_MIN_TRADES_DEFAULT = 1
# 웹 Optuna: min_trades = 거래일 수 × 하루 최소건 (소수 잭팟 컷)
OPTUNA_MIN_TRADES_PER_DAY_DEFAULT = 2
# 꼬리: 거래 빈도 낮음 — 탐색 min_trades 고정 (기간×일수 대신)
OPTUNA_TAIL_MIN_TRADES_DEFAULT = 1
# 새 score: PnL/(MDD+ADD) × √(min(n,soft_n)/soft_n)
OPTUNA_SCORE_MDD_ADD_DEFAULT = 10000.0
# 구 score: PnL / max(MDD, FLOOR) — sort_by=score_legacy
OPTUNA_SCORE_MDD_FLOOR_DEFAULT = 10000.0
# soft_n 기본 = 하루최소건 × 일수(기본 2) — 짧은 구간에서 15 고정이 과함
OPTUNA_SCORE_TRADE_SOFT_DAYS_DEFAULT = 2
OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT = 40.0
OPTUNA_REPORT_MIN_PF_DEFAULT = 1.0
OPTUNA_SORT_BY_DEFAULT = "score"
OPTUNA_SORT_BY_CHOICES = ("score", "score_legacy", "pnl", "daily_avg", "win_rate")
OPTUNA_WEB_SORT_BY_CHOICES = ("score", "score_legacy", "pnl", "daily_avg")

# 일별 손익 안정성 티어 (results_stable) — 학습1위/gated 와 별도 후보
# 손실일·최악일·일PnL 분산으로 “합산만 큰” 후보를 걸러낸다.
OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT = 1
OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT = -30000.0
OPTUNA_STABLE_LAMBDA_DEFAULT = 1.0
OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT = 2


def optuna_search_gate_defaults() -> Tuple[float, float, int]:
    """탐색 중 objective 게이트 (기본 0/0/1). CLI 미지정 시 사용."""
    return (
        float(get_env_float("PARAM_SEARCH_OPTUNA_MIN_WIN_RATE", OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT)),
        float(get_env_float("PARAM_SEARCH_OPTUNA_MIN_PF", OPTUNA_SEARCH_MIN_PF_DEFAULT)),
        int(get_env_int("PARAM_SEARCH_OPTUNA_MIN_TRADES", OPTUNA_SEARCH_MIN_TRADES_DEFAULT)),
    )


def optuna_min_trades_per_day(strategy: Optional[str] = None) -> int:
    """기간 자동 min_trades 의 하루 기준 건수 (기본 2). 꼬리는 resolve 에서 별도."""
    return max(1, int(get_env_int(
        "OPTUNA_MIN_TRADES_PER_DAY", OPTUNA_MIN_TRADES_PER_DAY_DEFAULT,
    )))


def optuna_tail_min_trades() -> int:
    """꼬리 Optuna 탐색 min_trades (기본 1 — 1주일이어도 후보 0건 방지)."""
    return max(1, int(get_env_int(
        "OPTUNA_TAIL_MIN_TRADES", OPTUNA_TAIL_MIN_TRADES_DEFAULT,
    )))


def resolve_optuna_min_trades(
    start: Any,
    end: Any,
    strategy: Optional[str] = None,
) -> Dict[str, Any]:
    """
    웹 Optuna용 min_trades.

    - tail: OPTUNA_TAIL_MIN_TRADES (기본 1, 기간 무관)
    - 그 외: max(1, 거래일수 × OPTUNA_MIN_TRADES_PER_DAY)
    CLI --min_trades 직접 지정 시 이 함수를 쓰지 않아도 됨.
    """
    from kis_trader.utils.kr_trading_day import count_kr_trading_days

    strat = str(strategy or "").strip().lower()
    n_days = count_kr_trading_days(start, end)
    if strat == "tail":
        min_tr = optuna_tail_min_trades()
        return {
            "min_trades": int(min_tr),
            "n_trading_days": int(n_days),
            "min_trades_per_day": 0,
            "min_trades_source": "tail_fixed",
        }
    per_day = optuna_min_trades_per_day(strat)
    min_tr = max(1, int(n_days) * int(per_day))
    return {
        "min_trades": int(min_tr),
        "n_trading_days": int(n_days),
        "min_trades_per_day": int(per_day),
        "min_trades_source": "period_auto",
    }


def annotate_optuna_period_daily_avg(out_data: Optional[Dict[str, Any]]) -> None:
    """결과 JSON 행에 기간 일평균 PnL(총손익÷거래일) 붙임. 활성일 mean 과 별개.

    2026-09-06: 결과 JSON 에 use_rust 필드도 함께 기록 (룰 29).
    → register_result_json_as_job 이 import 시 뱃지 표기 정합 (❔ 재발 방지).
    """
    if not isinstance(out_data, dict):
        return
    # use_rust: BACKTEST_USE_RUST 환경변수 기준. 이미 기록돼 있으면 유지 (덮어쓰기 금지).
    if "use_rust" not in out_data:
        import os as _os
        out_data["use_rust"] = bool(_os.environ.get("BACKTEST_USE_RUST") == "1")
    from kis_trader.utils.kr_trading_day import count_kr_trading_days

    start = out_data.get("start")
    end = out_data.get("end")
    try:
        n_days = int(out_data.get("n_trading_days") or 0)
    except (TypeError, ValueError):
        n_days = 0
    if n_days <= 0 and start and end:
        n_days = count_kr_trading_days(start, end)
    n_days = max(1, int(n_days or 1))
    out_data["n_trading_days"] = n_days
    if out_data.get("min_trades_per_day") is None:
        out_data["min_trades_per_day"] = optuna_min_trades_per_day()
    try:
        budget = float(
            out_data.get("total_budget_krw")
            or out_data.get("total_budget")
            or 0
        )
    except (TypeError, ValueError):
        budget = 0.0
    keys = (
        "results", "results_all", "results_gated", "results_stable",
        "results_mode", "mode_combo_results",
    )
    for key in keys:
        rows = out_data.get(key)
        if not isinstance(rows, list):
            continue
        for r in rows:
            if not isinstance(r, dict):
                continue
            try:
                pnl = float(r.get("total_pnl") or 0)
            except (TypeError, ValueError):
                pnl = 0.0
            r["n_period_trading_days"] = n_days
            r["period_daily_avg_pnl"] = round(pnl / float(n_days), 2)
            if budget > 0:
                r["period_daily_avg_pct"] = round(
                    pnl / budget * 100.0 / float(n_days), 3,
                )
            elif r.get("daily_avg_pct") is not None:
                r["period_daily_avg_pct"] = r.get("daily_avg_pct")
            elif r.get("bot_pct") is not None:
                try:
                    r["period_daily_avg_pct"] = round(
                        float(r["bot_pct"]) / float(n_days), 3,
                    )
                except (TypeError, ValueError):
                    pass


def optuna_score_mdd_add() -> float:
    """새 score 분모 MDD+ADD 의 ADD (기본 10000원)."""
    return max(1.0, float(get_env_float(
        "OPTUNA_SCORE_MDD_ADD", OPTUNA_SCORE_MDD_ADD_DEFAULT,
    )))


def optuna_score_mdd_floor(strategy: Optional[str] = None) -> float:
    """
    구 score 분모 하한 max(MDD, floor).

    전략별 {PREFIX}_SCORE_MDD_FLOOR 가 있으면 우선, 없으면 OPTUNA_SCORE_MDD_FLOOR.
    """
    prefix_map = {
        "tail": "TAIL",
        "momentum": "MOMENTUM",
        "us_momentum": "US_MOMENTUM",
        "breakout": "BREAKOUT",
        "scalp": "SCALP",
    }
    strat = str(strategy or "").strip().lower()
    prefix = prefix_map.get(strat)
    if prefix:
        raw = str(get_env_from_db(f"{prefix}_SCORE_MDD_FLOOR", "") or "").strip()
        if raw:
            try:
                return max(1.0, float(raw))
            except (TypeError, ValueError):
                pass
    return max(1.0, float(get_env_float(
        "OPTUNA_SCORE_MDD_FLOOR", OPTUNA_SCORE_MDD_FLOOR_DEFAULT,
    )))


def optuna_score_trade_soft_n() -> int:
    """
    새 score 거래수 soft 포화점.

    OPTUNA_SCORE_TRADE_SOFT_N 이 있으면 그 값.
    없으면 OPTUNA_MIN_TRADES_PER_DAY × OPTUNA_SCORE_TRADE_SOFT_DAYS(기본 2)
    → 하루 2건 × 2일 = 4 (짧은 구간에서 15 고정 과감점 방지).
    """
    raw = str(get_env_from_db("OPTUNA_SCORE_TRADE_SOFT_N", "") or "").strip()
    if raw:
        try:
            return max(1, int(float(raw)))
        except (TypeError, ValueError):
            pass
    days = max(1, int(get_env_int(
        "OPTUNA_SCORE_TRADE_SOFT_DAYS", OPTUNA_SCORE_TRADE_SOFT_DAYS_DEFAULT,
    )))
    return max(1, int(optuna_min_trades_per_day()) * int(days))


def normalize_optuna_sort_by(sort_by: Any, *, web: bool = False) -> str:
    """sort_by 정규화. 웹은 score|pnl|daily_avg 만."""
    sb = str(sort_by or "").strip().lower()
    if sb in ("score_v2", "risk_score"):
        sb = "score"
    if sb in ("legacy", "score_v1", "score_floor", "pnl_mdd"):
        sb = "score_legacy"
    if sb in ("period_daily_avg", "daily", "avg_daily"):
        sb = "daily_avg"
    if sb in ("stability", "stable"):
        return "stability"
    allowed = OPTUNA_WEB_SORT_BY_CHOICES if web else OPTUNA_SORT_BY_CHOICES
    if not sb or sb not in allowed:
        return OPTUNA_SORT_BY_DEFAULT
    return sb


def optuna_objective_value(
    result: Optional[Dict[str, Any]],
    sort_by: str = "score",
    *,
    start: Any = None,
    end: Any = None,
    n_trading_days: Optional[int] = None,
    strategy: Optional[str] = None,
) -> float:
    """
    Optuna 목적함수 (maximize).

    - score: (PnL / (MDD + ADD)) × √(min(trades, soft_n) / soft_n)
    - score_legacy: PnL / max(MDD, FLOOR) — (구) 순익/MDD하한
    - daily_avg: PnL ÷ 기간 한국거래일
    - pnl: 총손익
    - win_rate: 승률 (CLI)
    """
    r = result if isinstance(result, dict) else {}
    sb = normalize_optuna_sort_by(sort_by, web=False)
    if sb == "stability":
        sb = OPTUNA_SORT_BY_DEFAULT
    try:
        pnl = float(r.get("total_pnl") or 0)
    except (TypeError, ValueError):
        pnl = 0.0
    if sb == "win_rate":
        try:
            return float(r.get("win_rate") or 0)
        except (TypeError, ValueError):
            return 0.0
    if sb == "pnl":
        return pnl
    if sb == "daily_avg":
        n = n_trading_days
        if n is None or int(n or 0) <= 0:
            if start is not None and end is not None:
                from kis_trader.utils.kr_trading_day import count_kr_trading_days
                n = count_kr_trading_days(start, end)
            else:
                n = 1
        return pnl / float(max(1, int(n)))
    if sb == "score_legacy":
        try:
            mdd = float(r.get("mdd") or 0)
        except (TypeError, ValueError):
            mdd = 0.0
        floor = optuna_score_mdd_floor(strategy)
        return pnl / max(mdd, floor)
    # score (수익·낙폭·표본)
    try:
        mdd = float(r.get("mdd") or 0)
    except (TypeError, ValueError):
        mdd = 0.0
    try:
        trades = float(r.get("total_trades") or 0)
    except (TypeError, ValueError):
        trades = 0.0
    add = optuna_score_mdd_add()
    soft_n = float(optuna_score_trade_soft_n())
    soft = (min(max(0.0, trades), soft_n) / soft_n) ** 0.5
    return (pnl / (max(0.0, mdd) + add)) * soft


def optuna_store_trial_score_user_attrs(
    trial: Any,
    result: Dict[str, Any],
    sort_by: str,
    *,
    start: Any = None,
    end: Any = None,
    strategy: Optional[str] = None,
) -> float:
    """trial score·score_legacy·일평균 저장 후 sort_by 목적값 반환."""
    kw = {"start": start, "end": end, "strategy": strategy}
    trial.set_user_attr(
        "score",
        float(optuna_objective_value(result, "score", **kw)),
    )
    trial.set_user_attr(
        "score_legacy",
        float(optuna_objective_value(result, "score_legacy", **kw)),
    )
    trial.set_user_attr(
        "period_daily_avg_pnl",
        float(optuna_objective_value(result, "daily_avg", **kw)),
    )
    return float(optuna_objective_value(result, sort_by, **kw))


def optuna_score_fields_from_trial(trial: Any) -> Dict[str, float]:
    """JSON 행용 score / score_legacy."""
    return {
        "score": float(trial.user_attrs.get("score") or 0),
        "score_legacy": float(trial.user_attrs.get("score_legacy") or 0),
    }


def optuna_report_gate_defaults() -> Tuple[float, float, int]:
    """결과 후보·apply 사후 필터 (기본 승률40·PF1.0·min_trades=탐색과 동일)."""
    _sw, _sp, min_tr = optuna_search_gate_defaults()
    return (
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_REPORT_MIN_WIN_RATE", OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT,
        )),
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_REPORT_MIN_PF", OPTUNA_REPORT_MIN_PF_DEFAULT,
        )),
        int(get_env_int("PARAM_SEARCH_OPTUNA_REPORT_MIN_TRADES", max(1, min_tr))),
    )


def _sort_optuna_rows(rows: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]:
    sb = normalize_optuna_sort_by(sort_by, web=False)
    out = list(rows)

    def _f(r: Dict[str, Any], k: str) -> float:
        try:
            return float(r.get(k) or 0)
        except (TypeError, ValueError):
            return 0.0

    if sb == "score":
        out.sort(key=lambda r: (-_f(r, "score"), -_f(r, "total_pnl"), -_f(r, "win_rate")))
    elif sb == "score_legacy":
        out.sort(
            key=lambda r: (
                -_f(r, "score_legacy" if r.get("score_legacy") is not None else "score"),
                -_f(r, "total_pnl"),
                -_f(r, "win_rate"),
            ),
        )
    elif sb == "daily_avg":
        def _avg_key(r: Dict[str, Any]) -> Tuple[float, float, float]:
            if r.get("period_daily_avg_pnl") is not None:
                avg = _f(r, "period_daily_avg_pnl")
            else:
                avg = _f(r, "daily_pnl_mean")
            return (-avg, -_f(r, "total_pnl"), -_f(r, "win_rate"))
        out.sort(key=_avg_key)
    elif sb == "win_rate":
        out.sort(key=lambda r: (-_f(r, "win_rate"), -_f(r, "total_pnl")))
    elif sb == "stability":
        # 일평균 − λ·표준편차(stability_score) 우선 · 최악일 · 합산 PnL
        out.sort(
            key=lambda r: (
                -_f(r, "stability_score"),
                -_f(r, "worst_day_pnl"),
                -_f(r, "total_pnl"),
                -_f(r, "win_rate"),
            ),
        )
    else:
        out.sort(key=lambda r: (-_f(r, "total_pnl"), -_f(r, "win_rate")))
    return out


def trade_exit_day_key(trade: Dict[str, Any]) -> str:
    """청산 시각 → YYYY-MM-DD (없으면 빈 문자열).

    꼬리 백테는 exit_time, 스캘핑·모멘텀·돌파 포트폴리오 백테는 sell_time 을 씀.
    sell_time 누락 시 daily_pnl/results_stable 이 전부 비게 됨.
    """
    raw = (
        trade.get("exit_time")
        or trade.get("sell_date")
        or trade.get("sell_time")  # scalp/momentum/breakout 포트폴리오
        or trade.get("exit_ts")
        or trade.get("exit_at")
        or ""
    )
    s = str(raw).strip()
    if not s:
        return ""
    digits = "".join(ch for ch in s if ch.isdigit())
    if len(digits) >= 8:
        return f"{digits[0:4]}-{digits[4:6]}-{digits[6:8]}"
    if len(s) >= 10 and s[4] == "-" and s[7] == "-":
        return s[:10]
    return ""


def compute_daily_stability_metrics(
    trades: List[Dict[str, Any]],
    *,
    stability_lambda: Optional[float] = None,
) -> Dict[str, Any]:
    """
    거래 리스트 → 일별 PnL·안정성 점수.

    stability_score = mean(일PnL) − λ × std(일PnL)
    (λ 기본 OPTUNA_STABLE_LAMBDA / get_env)
    """
    from statistics import mean, pstdev

    if stability_lambda is None:
        _, _, lam, _ = optuna_stable_gate_defaults()
        stability_lambda = lam
    try:
        lam = float(stability_lambda)
    except (TypeError, ValueError):
        lam = float(OPTUNA_STABLE_LAMBDA_DEFAULT)

    by_day: Dict[str, float] = {}
    for t in trades or []:
        day = trade_exit_day_key(t if isinstance(t, dict) else {})
        if not day:
            continue
        try:
            pnl = float((t or {}).get("pnl") or (t or {}).get("realized_pnl") or 0)
        except (TypeError, ValueError):
            pnl = 0.0
        by_day[day] = by_day.get(day, 0.0) + pnl

    days_sorted = sorted(by_day.keys())
    vals = [float(by_day[d]) for d in days_sorted]
    n_days = len(vals)
    if n_days <= 0:
        return {
            "daily_pnl": {},
            "n_active_days": 0,
            "n_losing_days": 0,
            "worst_day_pnl": 0.0,
            "best_day_pnl": 0.0,
            "daily_pnl_mean": 0.0,
            "daily_pnl_std": 0.0,
            "stability_score": 0.0,
            "stability_lambda": lam,
        }

    n_lose = sum(1 for v in vals if v < 0)
    worst = min(vals)
    best = max(vals)
    avg = float(mean(vals))
    std = float(pstdev(vals)) if n_days >= 2 else 0.0
    score = avg - lam * std
    return {
        "daily_pnl": {d: round(by_day[d], 2) for d in days_sorted},
        "n_active_days": n_days,
        "n_losing_days": int(n_lose),
        "worst_day_pnl": round(worst, 2),
        "best_day_pnl": round(best, 2),
        "daily_pnl_mean": round(avg, 2),
        "daily_pnl_std": round(std, 2),
        "stability_score": round(score, 4),
        "stability_lambda": lam,
    }


def attach_daily_stability(
    result: Dict[str, Any],
    trades: List[Dict[str, Any]],
) -> Dict[str, Any]:
    """evaluate_* 반환 dict 에 일별 안정성 필드를 붙인다."""
    if not isinstance(result, dict):
        return result
    result.update(compute_daily_stability_metrics(trades or []))
    return result


def attach_optional_backtest_trades(
    result: Dict[str, Any],
    trades: List[Dict[str, Any]],
    include_trades: bool = False,
) -> Dict[str, Any]:
    """Optuna 후처리용. include_trades=False 면 기존과 동일(JSON/trial attrs 비대화 방지)."""
    if include_trades and isinstance(result, dict):
        result["_trades"] = list(trades or [])
    return result


def slim_trades_for_optuna_json(
    trades: Optional[List[Dict[str, Any]]],
) -> List[Dict[str, Any]]:
    """Optuna 결과 JSON용 체결 요약 — 정합 diff용 최소 필드만 (전체 봉/틱 메타 제외)."""
    out: List[Dict[str, Any]] = []
    for t in trades or []:
        if not isinstance(t, dict):
            continue
        out.append(
            {
                "code": t.get("code") or t.get("ticker"),
                "buy_time": t.get("buy_time") or t.get("entry_time"),
                "sell_time": t.get("sell_time") or t.get("exit_time"),
                "pnl": t.get("pnl"),
                "sell_reason": (
                    t.get("sell_reason") or t.get("reason") or t.get("exit_reason")
                ),
                "entry_price": t.get("entry_price") or t.get("buy_price"),
                "exit_price": t.get("exit_price") or t.get("sell_price"),
                "qty": t.get("qty") or t.get("quantity"),
            }
        )
    return out


def optuna_stable_gate_defaults() -> Tuple[int, float, float, int]:
    """(max_losing_days, min_worst_day_pnl, lambda, min_active_days)."""
    return (
        int(get_env_int(
            "PARAM_SEARCH_OPTUNA_STABLE_MAX_LOSING_DAYS",
            OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT,
        )),
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_STABLE_MIN_WORST_DAY_PNL",
            OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT,
        )),
        float(get_env_float(
            "PARAM_SEARCH_OPTUNA_STABLE_LAMBDA",
            OPTUNA_STABLE_LAMBDA_DEFAULT,
        )),
        int(get_env_int(
            "PARAM_SEARCH_OPTUNA_STABLE_MIN_ACTIVE_DAYS",
            OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT,
        )),
    )


def row_passes_report_gates(
    row: Dict[str, Any],
    *,
    min_win_rate: float,
    min_pf: float,
    min_trades: int,
) -> bool:
    try:
        wr = float(row.get("win_rate") or 0)
        pf = float(row.get("pf") or 0)
        nt = int(row.get("total_trades") or 0)
    except (TypeError, ValueError):
        return False
    if nt < int(min_trades):
        return False
    if wr < float(min_win_rate):
        return False
    if pf < float(min_pf):
        return False
    return True


def row_passes_stable_gates(row: Dict[str, Any]) -> bool:
    """
    일별 안정성 사후 게이트.
    daily_pnl / n_active_days 가 없으면(구 JSON) 통과 불가 → results_stable 빈 목록.
    """
    if row.get("daily_pnl") is None and row.get("n_active_days") is None:
        return False
    max_lose, min_worst, _lam, min_days = optuna_stable_gate_defaults()
    try:
        n_days = int(row.get("n_active_days") or 0)
        n_lose = int(row.get("n_losing_days") or 0)
        worst = float(row.get("worst_day_pnl") or 0)
    except (TypeError, ValueError):
        return False
    if n_days < int(min_days):
        return False
    if n_lose > int(max_lose):
        return False
    if worst < float(min_worst):
        return False
    return True


def set_optuna_trial_stability_attrs(trial: Any, result: Dict[str, Any]) -> None:
    """Optuna trial.user_attrs 에 일별 안정성 스냅샷 저장."""
    import json as _json

    if not result:
        return
    try:
        trial.set_user_attr("n_active_days", int(result.get("n_active_days") or 0))
        trial.set_user_attr("n_losing_days", int(result.get("n_losing_days") or 0))
        trial.set_user_attr("worst_day_pnl", float(result.get("worst_day_pnl") or 0))
        trial.set_user_attr("best_day_pnl", float(result.get("best_day_pnl") or 0))
        trial.set_user_attr("daily_pnl_mean", float(result.get("daily_pnl_mean") or 0))
        trial.set_user_attr("daily_pnl_std", float(result.get("daily_pnl_std") or 0))
        trial.set_user_attr("stability_score", float(result.get("stability_score") or 0))
        trial.set_user_attr(
            "daily_pnl_json",
            _json.dumps(result.get("daily_pnl") or {}, ensure_ascii=False),
        )
    except Exception:
        pass


def stability_fields_from_trial_attrs(trial: Any) -> Dict[str, Any]:
    """trial.user_attrs → 결과 row 안정성 필드."""
    import json as _json

    raw = trial.user_attrs.get("daily_pnl_json") or "{}"
    try:
        daily = _json.loads(raw) if isinstance(raw, str) else (raw or {})
    except Exception:
        daily = {}
    if trial.user_attrs.get("n_active_days") is None and not daily:
        return {}
    return {
        "daily_pnl": daily if isinstance(daily, dict) else {},
        "n_active_days": int(trial.user_attrs.get("n_active_days") or 0),
        "n_losing_days": int(trial.user_attrs.get("n_losing_days") or 0),
        "worst_day_pnl": float(trial.user_attrs.get("worst_day_pnl") or 0),
        "best_day_pnl": float(trial.user_attrs.get("best_day_pnl") or 0),
        "daily_pnl_mean": float(trial.user_attrs.get("daily_pnl_mean") or 0),
        "daily_pnl_std": float(trial.user_attrs.get("daily_pnl_std") or 0),
        "stability_score": float(trial.user_attrs.get("stability_score") or 0),
    }


def build_results_stable_tier(
    rows: List[Dict[str, Any]],
    *,
    top_n: int = 10,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    """
    안정 Top — 사후합격(gated)·플러스 PnL 과 독립 (마이너스도 상대 순위).

    1) 일별 안정 게이트 통과분 → 안정점수순
    2) 0건이면 학습풀 전체를 안정점수순 TopN (게이트는 참고·폴백 표시)
       → 3일 전패장에서 max_losing_days=1 이면 게이트 0이어도 표가 비지 않음
    """
    max_lose, min_worst, lam, min_days = optuna_stable_gate_defaults()
    n = max(1, int(top_n or 10))
    all_rows = [r for r in (rows or []) if isinstance(r, dict)]
    pool = [r for r in all_rows if row_passes_stable_gates(r)]
    fallback = False
    if pool:
        stable = _sort_optuna_rows(pool, "stability")[:n]
    else:
        fallback = True
        with_stab = [
            r for r in all_rows
            if r.get("stability_score") is not None
            or r.get("n_active_days") is not None
            or r.get("daily_pnl") is not None
        ]
        src = with_stab if with_stab else all_rows
        stable = _sort_optuna_rows(src, "stability")[:n]
    meta = {
        "max_losing_days": max_lose,
        "min_worst_day_pnl": min_worst,
        "stability_lambda": lam,
        "min_active_days": min_days,
        "score_note": "stability_score = mean(일PnL) − λ × std(일PnL)",
        "fallback_rank_only": bool(fallback),
        "n_gate_pass": len(pool),
        "fallback_note": (
            "안정 게이트 0건 → 학습풀 안정점수순 TopN (마이너스 PnL 포함 · 상대비교)"
            if fallback else ""
        ),
    }
    return stable, meta


def resolve_results_stable(
    data: Optional[Dict[str, Any]],
    *,
    top_n: Optional[int] = None,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    """JSON results_stable 우선 · 비면 학습풀에서 즉시 재구성 (구잡·gated=0 공용)."""
    data = data or {}
    try:
        n = int(top_n) if top_n is not None else 10
    except (TypeError, ValueError):
        n = 10
    n = max(1, n)
    stored = [r for r in list(data.get("results_stable") or []) if isinstance(r, dict)]
    gates = dict(data.get("stable_gates") or {})
    if stored:
        return stored[:n], gates
    allr = list(data.get("results_all") or data.get("results") or [])
    stable, meta = build_results_stable_tier(allr, top_n=n)
    gates.update(meta)
    return stable, gates


def resolve_results_mode_consensus(
    data: Optional[Dict[str, Any]],
    *,
    top_n: Optional[int] = None,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    """JSON results_mode 우선 · 없으면 mode Top10 즉시 재구성 (구 JSON 호환)."""
    data = data or {}
    try:
        n = int(top_n) if top_n is not None else 10
    except (TypeError, ValueError):
        n = 10
    n = max(1, n)
    stored = [r for r in list(data.get("results_mode") or []) if isinstance(r, dict)]
    meta = dict(data.get("mode_consensus_meta") or {})
    if stored:
        return stored[:n], meta
    allr = list(data.get("results_all") or data.get("results") or [])
    from kis_trader.backtest.optuna_mode_combo import build_results_mode_consensus_tier

    rows, built_meta = build_results_mode_consensus_tier(
        allr,
        top_n=n,
        grid_keys=list(data.get("grid_keys") or []),
        data=data,
    )
    meta.update(built_meta)
    return rows, meta


def build_optuna_result_tiers(
    rows: List[Dict[str, Any]],
    *,
    sort_by: str,
    top_n: int = 5000,
) -> Dict[str, Any]:
    """
    탐색 전체 vs 리포트/apply 후보 분리.

    - results_all: 완료·게이트통과(탐색게이트) trial 전부 정렬
    - results: 하위호환 — 플러스 PnL 우선(없으면 all)
    - results_gated: 승률·PF 사후 필터 (apply 후보, PnL>0)
    - results_stable: 학습풀 일별 안정성 (gated·플러스 독립 · 게이트0이면 점수순 폴백)
    """
    rep_wr, rep_pf, rep_tr = optuna_report_gate_defaults()
    all_sorted = _sort_optuna_rows(rows, sort_by)
    profitable = [r for r in all_sorted if float(r.get("total_pnl") or 0) > 0]
    learning = profitable if profitable else all_sorted
    gated = [
        r for r in all_sorted
        if row_passes_report_gates(
            r, min_win_rate=rep_wr, min_pf=rep_pf, min_trades=rep_tr,
        )
        and float(r.get("total_pnl") or 0) > 0
    ]
    # 안정 TopN 표용 — 전체 풀에서 상위 (gated 잘림·플러스와 무관)
    try:
        stable_ui_n = max(1, int(get_env_int("OPTUNA_POST_TOP_N", 10)))
    except Exception:
        stable_ui_n = 10
    stable, stable_meta = build_results_stable_tier(all_sorted, top_n=stable_ui_n)
    return {
        "results_all": all_sorted[:top_n],
        "results": learning[:top_n],
        "results_gated": gated[:top_n],
        "results_stable": stable,
        "report_gates": {
            "min_win_rate": rep_wr,
            "min_pf": rep_pf,
            "min_trades": rep_tr,
        },
        "stable_gates": stable_meta,
        "search_gates_note": (
            "탐색 min_win_rate/min_pf 기본 0 — TPE가 PnL 차이를 학습. "
            "적용·운영 후보는 results_gated(report_gates). "
            "들쭉날쭉 완화·상대비교는 results_stable (gated/플러스 독립 · 게이트0이면 점수순 폴백)."
        ),
        "n_results_all": len(all_sorted),
        "n_results_learning": len(learning),
        "n_results_gated": len(gated),
        "n_results_stable": len(stable),
    }


def _optuna_overfit_sample_days(data: Dict[str, Any]) -> int:
    try:
        days = int(data.get("backtest_days") or 0)
    except (TypeError, ValueError):
        days = 0
    if days > 0:
        return days
    start = str(data.get("start") or "")
    end = str(data.get("end") or "")
    try:
        from datetime import datetime as _dt

        return max(
            1,
            (_dt.strptime(end, "%Y-%m-%d") - _dt.strptime(start, "%Y-%m-%d")).days + 1,
        )
    except Exception:
        return 1


def overfit_risk_pct_for_row(
    data: Dict[str, Any],
    row: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
    """
    후보 한 줄의 과적합 가능도% (0~100, 높을수록 위험 · 만점=100).

    스터디 공통(표본 장일) + 이 trial의 거래수·승률·PF 이상치.
    교차검증이 아님. DB apply 게이트와 별개.
    """
    def _f(x: Any, default: float = 0.0) -> float:
        try:
            return float(x)
        except (TypeError, ValueError):
            return default

    def _i(x: Any, default: int = 0) -> int:
        try:
            return int(x)
        except (TypeError, ValueError):
            return default

    days = _optuna_overfit_sample_days(data or {})
    risk = 0.0
    # 2026-09-06 Y안 (룰 19 사용자 선택): 표본 일수 max 40 → 25 감소.
    # 근거: 다일 확보가 어려운 개발 초기·특정 종목 대응·리허설 백테에서 견고성 지표(팩터 5·6)
    # 로 상쇄 가능해야. 이전엔 1일=자동 40점+ → 진짜 견고한 조합도 비권장 뜨는 부작용.
    if days <= 1:
        risk += 25.0
    elif days == 2:
        risk += 18.0
    elif days <= 4:
        risk += 10.0

    if not row:
        risk = max(0.0, min(100.0, round(risk + 25.0, 1)))
        return {
            "overfit_risk_pct": risk,
            "verdict": "비권장",
            "verdict_ui": "위험 · 비권장",
        }

    nt = _i(row.get("total_trades"))
    wr = _f(row.get("win_rate"))
    pf = _f(row.get("pf"))
    pnl = _f(row.get("total_pnl"))

    if nt <= 1:
        risk += 25.0
    elif nt <= 3:
        risk += 18.0
    elif nt <= 9:
        risk += 10.0

    if wr >= 90.0 and nt < 10:
        risk += 15.0
    elif wr >= 80.0 and nt < 5:
        risk += 10.0
    if pf >= 50.0 and nt < 10:
        risk += 10.0
    elif pf >= 10.0 and nt < 5:
        risk += 6.0

    gated = list((data or {}).get("results_gated") or [])
    learn = list((data or {}).get("results") or (data or {}).get("results_all") or [])
    pool = gated if gated else learn
    if pool and nt > 0:
        best_pnl = round(pnl, 0)
        same = [r for r in pool if abs(_f(r.get("total_pnl")) - best_pnl) < 1.0]
        share = len(same) / max(1, len(pool))
        if share >= 0.4 and len(same) >= 5:
            risk += 12.0
        elif share >= 0.25 and len(same) >= 3:
            risk += 6.0

    risk = max(0.0, min(100.0, round(risk, 1)))
    if risk >= 70.0:
        verdict, verd_ui = "비권장", "위험 · 비권장"
    elif risk >= 40.0:
        verdict, verd_ui = "주의", "주의"
    else:
        verdict, verd_ui = "상대적으로낮음", "상대적으로 낮음"
    return {
        "overfit_risk_pct": risk,
        "verdict": verdict,
        "verdict_ui": verd_ui,
    }


def build_optuna_overfit_diagnostics(data: Dict[str, Any]) -> Dict[str, Any]:
    """
    Optuna 결과 → 과적합 위험% · 적용 가능도% · 임계값(파라미터) 분포 표용 dict.

    - 통계적 교차검증이 아니라 **운영 휴리스틱**(표본 일수·거래수·승률/PF 이상치·평탄 고원).
    - 높을수록 과적합 위험. 적용 가능도 ≈ 100 − 위험 (하한 0).
    - 웹·브리핑·JSON 공통. DB apply 게이트는 바꾸지 않음(표시·판별용).
    """
    import statistics

    def _f(x: Any, default: float = 0.0) -> float:
        try:
            return float(x)
        except (TypeError, ValueError):
            return default

    def _i(x: Any, default: int = 0) -> int:
        try:
            return int(x)
        except (TypeError, ValueError):
            return default

    days = _i(data.get("backtest_days"), 0)
    if days <= 0:
        start = str(data.get("start") or "")
        end = str(data.get("end") or "")
        try:
            from datetime import datetime as _dt

            days = max(
                1,
                (_dt.strptime(end, "%Y-%m-%d") - _dt.strptime(start, "%Y-%m-%d")).days + 1,
            )
        except Exception:
            days = 1

    gated = list(data.get("results_gated") or [])
    learn = list(data.get("results") or data.get("results_all") or [])
    pool = gated if gated else learn
    top = pool[0] if pool else None
    n_gated = _i(data.get("n_results_gated"), len(gated))
    n_all = _i(data.get("n_results_all"), len(data.get("results_all") or learn))
    n_stable = _i(data.get("n_results_stable"), len(data.get("results_stable") or []))

    factors: List[Dict[str, Any]] = []
    risk = 0.0

    # 1) 표본 장일 (2026-09-06 Y안 · 룰 19: max 40 → 25 감소)
    # 다일 확보 어려운 개발/리허설·특정 종목 백테는 견고성 지표(팩터 6)로 상쇄 가능해야 함.
    if days <= 1:
        pts, detail = 25.0, f"거래일≈{days}일 — 단일 장 표본 부족(견고성으로 상쇄 필요)"
    elif days == 2:
        pts, detail = 18.0, f"거래일≈{days}일 — 이틀만으로는 추세 전환에 취약"
    elif days <= 4:
        pts, detail = 10.0, f"거래일≈{days}일 — 다일 재검증 권장(≥5일)"
    else:
        pts, detail = 0.0, f"거래일≈{days}일 — 표본 일수 상대적 양호"
    risk += pts
    factors.append({"id": "sample_days", "label": "표본 장일", "points": pts, "detail": detail})

    nt = _i(top.get("total_trades")) if top else 0
    wr = _f(top.get("win_rate")) if top else 0.0
    pf = _f(top.get("pf")) if top else 0.0
    pnl = _f(top.get("total_pnl")) if top else 0.0

    # 2) 거래 표본
    if not top:
        pts, detail = 25.0, "통과 후보 없음 — 적용 불가"
    elif nt <= 1:
        pts, detail = 25.0, f"상위 후보 거래 {nt}건 — 우연 승·과적합 가능"
    elif nt <= 3:
        pts, detail = 18.0, f"상위 후보 거래 {nt}건 — 표본 부족"
    elif nt <= 9:
        pts, detail = 10.0, f"상위 후보 거래 {nt}건 — 해석 시 주의"
    else:
        pts, detail = 0.0, f"상위 후보 거래 {nt}건 — 상대적 양호"
    risk += pts
    factors.append({"id": "trade_count", "label": "거래 표본", "points": pts, "detail": detail})

    # 3) 승률/PF 이상치
    pts = 0.0
    bits: List[str] = []
    if top and wr >= 90.0 and nt < 10:
        pts += 15.0
        bits.append(f"승률 {wr:.1f}% + 거래 {nt}건")
    elif top and wr >= 80.0 and nt < 5:
        pts += 10.0
        bits.append(f"승률 {wr:.1f}% + 거래 {nt}건")
    if top and pf >= 50.0 and nt < 10:
        pts += 10.0
        bits.append(f"PF {pf:.2f} (소수 거래 폭증)")
    elif top and pf >= 10.0 and nt < 5:
        pts += 6.0
        bits.append(f"PF {pf:.2f}")
    detail = " · ".join(bits) if bits else "이상치 없음"
    risk += pts
    factors.append({"id": "outlier_wr_pf", "label": "승률·PF 이상치", "points": pts, "detail": detail})

    # 4) gated 부재 / 거의 전원 통과
    pts = 0.0
    if n_gated <= 0 and n_all > 0:
        pts = 12.0
        detail = f"사후합격 0건 (학습 {n_all}) — DB 적용 비권장"
    elif n_all > 0 and n_gated / max(1, n_all) >= 0.85 and days <= 2:
        pts = 10.0
        detail = f"gated/all={n_gated}/{n_all} — 단일에 대부분 통과(필터 느슨·노이즈)"
    elif n_gated > 0:
        pts = 0.0
        detail = f"사후합격 {n_gated}건 · stable {n_stable}건"
    else:
        pts = 8.0
        detail = "학습·gated 모두 비어 있음"
    risk += pts
    factors.append({"id": "gate_coverage", "label": "게이트 커버", "points": pts, "detail": detail})

    # 5) PnL 고원(동일 best 반복) — 2026-09-06 Y안 재해석 (룰 19 사용자 선택)
    # 기존: 동일 PnL 반복 = 무조건 위험 (+12점)
    # 정정: 동일 PnL 반복 & **파라미터도 좁음** = TPE 좁게 튐 (여전히 위험 · 최대 +12)
    #       동일 PnL 반복 & **파라미터 다양** = 파라미터 민감도 낮음 = 견고 (감점 -8)
    # 판단: same pool 에서 핵심 파라미터(tp_pct/sl_pct/drop_rate/vol_mult) 의
    #       고유값 개수 대비 표본 크기 비율(unique_ratio)로 근사.
    plateau_share = 0.0
    plateau_n = 0
    plateau_param_diverse = False
    if pool and top:
        best_pnl = round(pnl, 0)
        same = [
            r for r in pool
            if abs(_f(r.get("total_pnl")) - best_pnl) < 1.0
        ]
        plateau_n = len(same)
        plateau_share = plateau_n / max(1, len(pool))
        # 핵심 파라미터의 고유값 다양성 (같은 PnL 이 여러 파라미터 조합에서 도달했나?)
        _core_keys = ("tp_pct", "sl_pct", "drop_rate", "vol_mult", "cooldown_min")
        _uniq_ratios: List[float] = []
        for k in _core_keys:
            vals = []
            for r in same:
                p = r.get("merged_params") or r.get("params") or {}
                if isinstance(p, dict) and k in p:
                    try:
                        vals.append(round(float(p[k]), 6))
                    except (TypeError, ValueError):
                        pass
            if len(vals) >= 3:
                _uniq_ratios.append(len(set(vals)) / len(vals))
        _avg_uniq = sum(_uniq_ratios) / max(1, len(_uniq_ratios)) if _uniq_ratios else 0.0
        # unique_ratio 0.5+ = 다양한 파라미터에서 같은 PnL 도달 = 견고
        plateau_param_diverse = _avg_uniq >= 0.5 and len(_uniq_ratios) >= 3

        if plateau_share >= 0.4 and plateau_n >= 5:
            if plateau_param_diverse:
                pts = -8.0
                detail = (
                    f"동일 PnL≈{best_pnl:,.0f}원이 {plateau_n}/{len(pool)} "
                    f"({plateau_share:.0%}) · 핵심 파라미터 다양성 {_avg_uniq:.0%} — 견고(-8)"
                )
            else:
                pts = 12.0
                detail = (
                    f"동일 PnL≈{best_pnl:,.0f}원이 {plateau_n}/{len(pool)} "
                    f"({plateau_share:.0%}) · 파라미터 좁음({_avg_uniq:.0%}) — TPE 몰빵/위험"
                )
        elif plateau_share >= 0.25 and plateau_n >= 3:
            if plateau_param_diverse:
                pts = -4.0
                detail = (
                    f"PnL 고원 {plateau_n}/{len(pool)} ({plateau_share:.0%}) · "
                    f"파라미터 다양({_avg_uniq:.0%}) — 소폭 견고(-4)"
                )
            else:
                pts = 6.0
                detail = f"PnL 고원 {plateau_n}/{len(pool)} ({plateau_share:.0%}) · 파라미터 좁음"
        else:
            pts = 0.0
            detail = f"고원 비율 {plateau_share:.0%} ({plateau_n}건)"
    else:
        pts, detail = 0.0, "고원 판정 스킵"
    risk += pts
    factors.append({"id": "pnl_plateau", "label": "PnL 고원 (파라미터 민감도)", "points": pts, "detail": detail})

    # 6) 파라미터 안정성 (2026-09-06 Y안 신설 · 사용자 지적: 녹색줄+주황점 일치=견고)
    # gated pool 전체에서 핵심 파라미터의 mode_share (최빈값 비율) 를 봄.
    # mode_share ≥ 0.6 인 파라미터가 여러 개면 = "여러 trial이 같은 값 선택" = 견고 → 감점
    # 판정: threshold_distribution 사전 계산 안 됐을 수 있어 pool 에서 직접 집계.
    core_keys_stab = (
        "tp_pct", "sl_pct", "drop_rate", "vol_mult", "cooldown_min",
        "shoulder_min_high", "shoulder_cut_pct", "high_chase_thr",
        "whipsaw_subbar_sec", "whipsaw_lookback_sec",
    )
    stab_stats: List[Tuple[str, float, int]] = []  # (key, mode_share, n)
    pool_for_stab = pool[: min(30, len(pool))] if pool else []
    if pool_for_stab:
        for k in core_keys_stab:
            vals: List[float] = []
            for r in pool_for_stab:
                p = r.get("merged_params") or r.get("params") or {}
                if isinstance(p, dict) and k in p:
                    try:
                        vals.append(round(float(p[k]), 6))
                    except (TypeError, ValueError):
                        pass
            if len(vals) >= 3:
                mode_v = max(set(vals), key=vals.count)
                mode_n = sum(1 for v in vals if v == mode_v)
                stab_stats.append((k, mode_n / len(vals), len(vals)))
    strong_stab = [s for s in stab_stats if s[1] >= 0.6]
    if len(strong_stab) >= 5:
        pts = -15.0
        detail = (
            f"핵심 파라미터 {len(strong_stab)}개가 mode_share≥60% — "
            f"다수 trial 동일값 선택 = 매우 견고(-15)"
        )
    elif len(strong_stab) >= 3:
        pts = -8.0
        detail = f"핵심 파라미터 {len(strong_stab)}개가 mode_share≥60% — 다소 견고(-8)"
    elif len(stab_stats) >= 3:
        pts = 0.0
        detail = (
            f"핵심 파라미터 {len(stab_stats)}개 집계, 견고 {len(strong_stab)}개 — 중립"
        )
    else:
        pts, detail = 0.0, "표본 부족 — 안정성 판정 스킵"
    risk += pts
    factors.append({"id": "param_stability", "label": "파라미터 안정성", "points": pts, "detail": detail})

    risk = max(0.0, min(100.0, round(risk, 1)))
    apply_pct = max(0.0, min(100.0, round(100.0 - risk, 1)))
    if risk >= 70.0:
        verdict = "비권장"
        verdict_ko = "과적합·표본부족 위험 높음 — 실매 DB 즉시 적용 비권장"
    elif risk >= 40.0:
        verdict = "주의"
        verdict_ko = "적용 가능도 중간 — 다일(≥5일) 재검증·웹백테 후 소액만"
    else:
        verdict = "상대적으로낮음"
        verdict_ko = "휴리스틱상 위험 상대적 낮음 — 그래도 다일 확인 권장"

    # --- 임계값 분포 (gated 우선, 상위 min(30, len) 행) ---
    dist_rows = pool[: min(30, len(pool))]
    skip_keys = {
        "params", "apply_cfg", "merged_params", "daily_pnl", "optuna_trial_number",
        "total_trades", "win_rate", "total_pnl", "pf", "score", "stability_score",
        "n_losing_days", "n_active_days", "worst_day_pnl", "best_day_pnl",
        "daily_pnl_mean", "daily_pnl_std", "skip_hts_scan_dupes",
    }
    prefer = list(data.get("grid_keys") or [])
    # 꼬리·공통에서 자주 보는 축
    prefer_extra = [
        "min_drop_rate", "min_recovery_ratio", "tail_ratio_min", "tail_pct_min",
        "stop_atr_mult", "target_atr_mult", "atr_sl_min_pct", "atr_sl_max_pct",
        "atr_tp_min_pct", "atr_tp_max_pct", "rsi_threshold", "bar_chg_min_pct",
        "bar_chg_max_pct", "shoulder_min_high", "shoulder_cut_pct", "cooldown_min",
        "max_daily", "whipsaw_filter_enabled", "ratchet_on", "sl_pct", "tp_pct",
    ]
    key_order = []
    for k in prefer + prefer_extra:
        if k not in key_order:
            key_order.append(k)

    # 실제 등장 키 수집
    value_maps: Dict[str, List[Any]] = {}
    for row in dist_rows:
        params = row.get("merged_params") or row.get("params") or {}
        if not isinstance(params, dict):
            continue
        for k, v in params.items():
            if k in skip_keys or str(k).startswith("_"):
                continue
            value_maps.setdefault(str(k), []).append(v)

    def _percentile(sorted_vals: List[float], p: float) -> float:
        if not sorted_vals:
            return 0.0
        if len(sorted_vals) == 1:
            return sorted_vals[0]
        idx = (len(sorted_vals) - 1) * p
        lo = int(idx)
        hi = min(lo + 1, len(sorted_vals) - 1)
        w = idx - lo
        return sorted_vals[lo] * (1.0 - w) + sorted_vals[hi] * w

    threshold_distribution: List[Dict[str, Any]] = []
    keys_out = [k for k in key_order if k in value_maps]
    # prefer 외 숫자 키 보충 (최대 18개 표시)
    for k in sorted(value_maps.keys()):
        if k not in keys_out:
            keys_out.append(k)
        if len(keys_out) >= 18:
            break

    for k in keys_out:
        vals = value_maps.get(k) or []
        if not vals:
            continue
        # bool / categorical
        as_num: List[float] = []
        for v in vals:
            if isinstance(v, bool):
                as_num.append(1.0 if v else 0.0)
            else:
                try:
                    as_num.append(float(v))
                except (TypeError, ValueError):
                    as_num = []
                    break
        # mode
        try:
            mode_v = statistics.mode(vals)
        except statistics.StatisticsError:
            mode_v = vals[0]
        mode_n = sum(1 for v in vals if v == mode_v)
        mode_share = mode_n / max(1, len(vals))
        row_d: Dict[str, Any] = {
            "param": k,
            "n": len(vals),
            "mode": mode_v,
            "mode_share": round(mode_share, 3),
        }
        if as_num:
            s = sorted(as_num)
            row_d["p25"] = round(_percentile(s, 0.25), 6)
            row_d["median"] = round(_percentile(s, 0.50), 6)
            row_d["p75"] = round(_percentile(s, 0.75), 6)
            row_d["min"] = round(s[0], 6)
            row_d["max"] = round(s[-1], 6)
        else:
            row_d["p25"] = None
            row_d["median"] = None
            row_d["p75"] = None
            row_d["min"] = None
            row_d["max"] = None
        threshold_distribution.append(row_d)

    pool_tag = "results_gated" if gated else "results(learning)"
    return {
        "overfit_risk_pct": risk,
        "apply_readiness_pct": apply_pct,
        "verdict": verdict,
        "verdict_ko": verdict_ko,
        "sample_days": days,
        "n_gated": n_gated,
        "n_all": n_all,
        "n_stable": n_stable,
        "top_trades": nt,
        "top_win_rate": wr,
        "top_pf": pf,
        "top_pnl": pnl,
        "plateau_share": round(plateau_share, 3),
        "plateau_n": plateau_n,
        "factors": factors,
        "threshold_distribution": threshold_distribution,
        "threshold_pool": pool_tag,
        "threshold_pool_n": len(dist_rows),
        "note": (
            "과적합%는 교차검증 점수가 아니라 표본·이상치·고원 휴리스틱입니다. "
            "적용 가능도%=100−과적합위험%. DB 적용 버튼 활성 조건(gated PnL>0)과는 별개입니다."
        ),
    }


def attach_optuna_overfit_diagnostics(data: Dict[str, Any]) -> Dict[str, Any]:
    """JSON dict 에 overfit_diagnostics 키를 채운다 (있으면 갱신)."""
    try:
        data["overfit_diagnostics"] = build_optuna_overfit_diagnostics(data)
    except Exception as exc:
        logger.warning("⚠️ overfit_diagnostics 생성 실패: %s", exc)
        data["overfit_diagnostics"] = {
            "overfit_risk_pct": None,
            "apply_readiness_pct": None,
            "verdict": "error",
            "verdict_ko": f"진단 실패: {exc}",
            "factors": [],
            "threshold_distribution": [],
            "note": str(exc),
        }
    return data


def pick_gated_apply_trial(
    study: Any,
    *,
    sort_by: str = "pnl",
    fail_objective: float = -1e18,
) -> Optional[Any]:
    """
    --apply-best 용: study.best(탐색 objective)가 아니라
    report_gates 통과 trial 중 정렬 1위.
    """
    import optuna  # noqa: WPS433 — 호출 시에만

    rep_wr, rep_pf, rep_tr = optuna_report_gate_defaults()
    cand: List[Tuple[Dict[str, Any], Any]] = []
    for trial in study.trials:
        if trial.state != optuna.trial.TrialState.COMPLETE:
            continue
        if not trial.user_attrs.get("gates_ok"):
            continue
        try:
            val = float(trial.value) if trial.value is not None else fail_objective
        except (TypeError, ValueError):
            val = fail_objective
        if val <= fail_objective + 1:
            continue
        row = {
            "win_rate": float(trial.user_attrs.get("win_rate") or 0),
            "pf": float(trial.user_attrs.get("pf") or 0),
            "total_trades": int(trial.user_attrs.get("total_trades") or 0),
            "total_pnl": float(trial.user_attrs.get("total_pnl") or 0),
            "score": float(trial.user_attrs.get("score") or 0),
            "_trial_number": int(trial.number),
        }
        if not row_passes_report_gates(
            row, min_win_rate=rep_wr, min_pf=rep_pf, min_trades=rep_tr,
        ):
            continue
        if float(row["total_pnl"]) <= 0:
            continue
        cand.append((row, trial))
    if not cand:
        return None
    ranked = _sort_optuna_rows([r for r, _ in cand], sort_by)
    top_n = int(ranked[0].get("_trial_number") or -1)
    for r, t in cand:
        if int(r.get("_trial_number") or -2) == top_n:
            return t
    return cand[0][1]


def ensure_optuna_gate_env_defaults(db: Any = None) -> None:
    """신규 Optuna 게이트 키가 DB에 없으면 env_config_ext 에만 UPSERT (전체 스냅샷 X)."""
    defaults = {
        "PARAM_SEARCH_OPTUNA_MIN_WIN_RATE": str(OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT),
        "PARAM_SEARCH_OPTUNA_MIN_PF": str(OPTUNA_SEARCH_MIN_PF_DEFAULT),
        "PARAM_SEARCH_OPTUNA_MIN_TRADES": str(OPTUNA_SEARCH_MIN_TRADES_DEFAULT),
        "OPTUNA_MIN_TRADES_PER_DAY": str(OPTUNA_MIN_TRADES_PER_DAY_DEFAULT),
        "OPTUNA_TAIL_MIN_TRADES": str(OPTUNA_TAIL_MIN_TRADES_DEFAULT),
        "OPTUNA_SCORE_MDD_ADD": str(int(OPTUNA_SCORE_MDD_ADD_DEFAULT)),
        "OPTUNA_SCORE_MDD_FLOOR": str(int(OPTUNA_SCORE_MDD_FLOOR_DEFAULT)),
        "OPTUNA_SCORE_TRADE_SOFT_DAYS": str(OPTUNA_SCORE_TRADE_SOFT_DAYS_DEFAULT),
        "PARAM_SEARCH_OPTUNA_REPORT_MIN_WIN_RATE": str(OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT),
        "PARAM_SEARCH_OPTUNA_REPORT_MIN_PF": str(OPTUNA_REPORT_MIN_PF_DEFAULT),
        "PARAM_SEARCH_OPTUNA_BRIEFING_AI": "1",
        # 일별 안정성 티어 (results_stable)
        "PARAM_SEARCH_OPTUNA_STABLE_MAX_LOSING_DAYS": str(OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT),
        "PARAM_SEARCH_OPTUNA_STABLE_MIN_WORST_DAY_PNL": str(OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT),
        "PARAM_SEARCH_OPTUNA_STABLE_LAMBDA": str(OPTUNA_STABLE_LAMBDA_DEFAULT),
        "PARAM_SEARCH_OPTUNA_STABLE_MIN_ACTIVE_DAYS": str(OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT),
        # Optuna apply 시 다단트레일 추천 → 전략별 *_DAILY_PROFIT_* (탐색 축 아님)
        "OPTUNA_DAILY_TRAIL_APPLY_ON_BEST": "true",
        "OPTUNA_DAILY_TRAIL_ARM_FRAC": "0.60",
        "OPTUNA_DAILY_TRAIL_BEST_FRAC": "0.70",
        "OPTUNA_DAILY_TRAIL_ARM_STEP": "5000",
        "OPTUNA_DAILY_TRAIL_MIN_ARM": "10000",
        "OPTUNA_DAILY_TRAIL_TIER_DROPS": "40,30,20",
        "OPTUNA_POST_TOP_N": "10",
        "OPTUNA_POST_INCLUDE_MODE": "true",
        "OPTUNA_POST_INCLUDE_LIVE": "true",
        "OPTUNA_POST_INCLUDE_STABLE": "true",
        "OPTUNA_POST_RUN_OB_WHIPSAW": "false",
        "OPTUNA_POST_FORCE_OB_WHIPSAW": "false",
        "OPTUNA_TPE_INCLUDE_ORDERBOOK": "true",
        "OPTUNA_TPE_INCLUDE_WHIPSAW": "true",
        "OPTUNA_OB_RECOMMEND_TRIALS": "500",
        "OPTUNA_OB_AXIS_TRIALS": "500",
        "OPTUNA_OB_COMBO_TRIALS_SINGLE": "150",
        "OPTUNA_OB_COMBO_TRIALS_DOUBLE": "200",
        "OPTUNA_OB_COMBO_TRIALS_TRIPLE": "250",
        "OPTUNA_WHIPSAW_PER_COMBO": "true",
        "OPTUNA_WHIPSAW_PER_COMBO_TRIALS": "100",
        "OPTUNA_WHIPSAW_PER_COMBO_MIN_TRADES": "3",
        "OPTUNA_OB_ENTRY_SPREAD_MIN": "0.1",
        "OPTUNA_OB_ENTRY_SPREAD_MAX": "8.0",
        "OPTUNA_OB_ENTRY_RATIO_MIN": "0.05",
        "OPTUNA_OB_ENTRY_RATIO_MAX": "1.5",
        "OPTUNA_OB_ENTRY_ASK_MULT_MIN": "1.0",
        "OPTUNA_OB_ENTRY_ASK_MULT_MAX": "80.0",
        "OPTUNA_OB_LOOKBACK_MIN": "30",
        "OPTUNA_OB_EXIT_HOLD_MIN": "1",
        "OPTUNA_OB_EXIT_HOLD_MAX": "5",
        "OPTUNA_OB_EXIT_RATIO_MIN": "0.2",
        "OPTUNA_OB_EXIT_RATIO_MAX": "0.8",
        "OPTUNA_OB_EXIT_PROFIT_MIN": "0.003",
        "OPTUNA_OB_EXIT_PROFIT_MAX": "0.02",
        "OPTUNA_OB_EXIT_MA_MIN": "3",
        "OPTUNA_OB_EXIT_MA_MAX": "10",
        "OPTUNA_OB_STOP_HOLD_MIN": "1",
        "OPTUNA_OB_STOP_HOLD_MAX": "5",
        "OPTUNA_OB_STOP_RATIO_MIN": "0.2",
        "OPTUNA_OB_STOP_RATIO_MAX": "0.8",
        "OPTUNA_OB_STOP_LOSS_MIN": "0.001",
        "OPTUNA_OB_STOP_LOSS_MAX": "0.02",
        "OPTUNA_OB_STOP_MA_MIN": "3",
        "OPTUNA_OB_STOP_MA_MAX": "10",
        "OPTUNA_WHIPSAW_RECOMMEND_TRIALS": "500",
        "OPTUNA_OB_HORIZON_MIN": "6",
        "OPTUNA_WHIPSAW_LOOKBACK_DAYS": "7",
        "OPTUNA_WHIPSAW_TICK_LOOKBACK_SEC": "180",
    }
    try:
        from datetime import datetime

        from database import TradeDB
    except ImportError:
        return
    owned = False
    if db is None:
        db = TradeDB()
        owned = True
    try:
        snap = db.get_merged_env_snapshot() or {}
        patch = {}
        for k, v in defaults.items():
            cur = snap.get(k)
            if cur is None or str(cur).strip() == "":
                patch[k] = v
        if not patch:
            return
        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        n = db._persist_env_config_overflow(patch, now)
        try:
            from kis_trader.utils.env import invalidate_merged_env_cache
            invalidate_merged_env_cache()
        except Exception:
            pass
        logger.info(
            "📌 Optuna 게이트 기본값 DB(ext) 반영 %d키: %s",
            n, sorted(patch.keys()),
        )
    except Exception as exc:
        logger.warning("⚠️ Optuna 게이트 기본값 DB 반영 실패: %s", exc)
    finally:
        if owned:
            try:
                db.conn.close()
            except Exception:
                pass


def mariadb_creds() -> dict:
    """TradeDB(database.py) 와 동일 우선순위 — env > 기본 141."""
    return {
        "host": os.environ.get("DB_HOST", "192.168.0.141"),
        "port": int(os.environ.get("DB_PORT", "3306")),
        "user": os.environ.get("DB_USER", "jae"),
        "password": os.environ.get("DB_PASS", "1234"),
    }


def resolve_optuna_db_name() -> str:
    """
    Optuna storage DB — 기본 kis_optuna (매매 kis_quant_db 와 분리).
    env OPTUNA_DB_NAME 로 오버라이드 가능.
    """
    raw = get_env_from_db("OPTUNA_DB_NAME", "")
    if raw and str(raw).strip() not in ("", "None"):
        return str(raw).strip()
    env = os.environ.get("OPTUNA_DB_NAME", "")
    if env and str(env).strip():
        return str(env).strip()
    return DEFAULT_OPTUNA_DB_NAME


def build_mariadb_storage_url(db_name: Optional[str] = None) -> str:
    """mysql+pymysql://…@141/optuna 형식 storage URL."""
    creds = mariadb_creds()
    name = (db_name or resolve_optuna_db_name()).strip()
    user = quote_plus(creds["user"])
    passwd = quote_plus(creds["password"])
    return (
        f"mysql+pymysql://{user}:{passwd}@{creds['host']}:{creds['port']}/{name}"
        f"?charset=utf8mb4"
    )


def ensure_optuna_database(db_name: Optional[str] = None) -> str:
    """
    MariaDB 141 — kis_optuna 존재 확인 (없으면 CREATE 시도).
    """
    name = (db_name or resolve_optuna_db_name()).strip()
    creds = mariadb_creds()

    try:
        import pymysql
    except ImportError as exc:
        raise ImportError(
            "Optuna MariaDB storage 는 pymysql 필요: pip install PyMySQL"
        ) from exc

    # DB 존재 여부만 확인 (이미 있으면 CREATE 생략)
    conn = pymysql.connect(
        host=creds["host"],
        port=creds["port"],
        user=creds["user"],
        password=creds["password"],
        charset="utf8mb4",
        autocommit=True,
        connect_timeout=10,
    )
    try:
        with conn.cursor() as cur:
            cur.execute("SHOW DATABASES LIKE %s", (name,))
            exists = cur.fetchone() is not None
            if not exists:
                cur.execute(
                    f"CREATE DATABASE IF NOT EXISTS `{name}` "
                    "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
                )
                logger.info(
                    "📦 Optuna DB 생성: %s@%s:%s/%s",
                    creds["user"], creds["host"], creds["port"], name,
                )
            else:
                logger.debug(
                    "📦 Optuna storage DB: %s@%s:%s/%s",
                    creds["user"], creds["host"], creds["port"], name,
                )
    except Exception as exc:
        logger.error("❌ Optuna DB '%s' 접속/확인 실패: %s", name, exc)
        raise
    finally:
        conn.close()
    return name


def resolve_optuna_storage_url(cli_override: Optional[str] = None) -> str:
    """
    Storage URL 우선순위:
      1) CLI --storage
      2) OPTUNA_STORAGE_URL (DB/env)
      3) MariaDB 141 / kis_optuna (TradeDB 동일 계정)
    """
    if cli_override and str(cli_override).strip():
        return str(cli_override).strip()
    from_db = get_env_from_db("OPTUNA_STORAGE_URL", "")
    if from_db and str(from_db).strip() not in ("", "None"):
        return str(from_db).strip()
    db_name = ensure_optuna_database()
    return build_mariadb_storage_url(db_name)


def resolve_study_name(
    *,
    strategy: str,
    mode: str,
    start: str,
    end: str,
    cli_override: Optional[str] = None,
    extra: Optional[str] = None,
) -> str:
    """Study 이름 — 전략·기간·모드 포함. extra=꼬리 진입모드 등(스터디 분리)."""
    if cli_override and str(cli_override).strip():
        return str(cli_override).strip()
    env_key = f"OPTUNA_{strategy.upper()}_STUDY_NAME"
    from_db = get_env_from_db(env_key, "")
    if from_db and str(from_db).strip() not in ("", "None"):
        return str(from_db).strip()
    legacy = get_env_from_db("OPTUNA_TAIL_STUDY_NAME", "")
    if strategy == "tail" and legacy and str(legacy).strip() not in ("", "None"):
        return str(legacy).strip()
    extra_s = str(extra or "").strip().lower()
    extra_s = f"_{extra_s}" if extra_s else ""
    return f"{strategy}_{mode}{extra_s}_{start}_{end}"


def optuna_run_lock_name(strategy: str) -> str:
    return f"{strategy}_param_search_optuna"


def release_shared_tick_store(ctx: Any, *, log: Optional[logging.Logger] = None) -> None:
    """
    Optuna ctx.shared_tick_store 해제.

    주의: ticks_by_code 가 공유메모리 뷰인 경우, unlink 이후 접근하면
    SIGBUS/강제종료(트레이스백 없음) 난다. 최빈(mode_combo) 실측·JSON 저장이
    끝난 뒤에만 호출할 것. optimize() 직후 즉시 unlink 금지.
    """
    lg = log or logger
    store = getattr(ctx, "shared_tick_store", None)
    if store is None:
        return
    try:
        store.unlink()
    except Exception as exc:
        lg.warning("⚠️ shared_tick_store unlink 실패: %s", exc)
    try:
        ctx.shared_tick_store = None
    except Exception:
        pass



def announce_optuna_json_path(
    out_path: str,
    *,
    strategy: str = "",
    mode: str = "",
    note: str = "",
    log: Optional[logging.Logger] = None,
) -> str:
    """
    결과 JSON 절대경로를 터미널·로그에 눈에 띄게 고지.
    또한 logs/optuna_<strategy>_<mode>_latest.jsonpath 에 기록 (없으면 strategy만).
    note 에 '최종' 이 포함되면 이전장/앞장 브리핑(.briefing.md) 생성.
    """
    abs_path = os.path.abspath(str(out_path or "").strip())
    lg = log or logger
    tag = note.strip() or "결과 JSON"
    line = f"📁 [{tag}] {abs_path}"
    # logger + print 이중 — nohup 로그·터미널 모두에서 바로 보이게
    lg.info("%s", line)
    print(line, flush=True)
    print(f"OPTUNA_RESULT_JSON={abs_path}", flush=True)

    try:
        root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
        logs_dir = os.path.join(root, "logs")
        os.makedirs(logs_dir, exist_ok=True)
        s = (strategy or "optuna").strip().lower() or "optuna"
        m = (mode or "run").strip().lower() or "run"
        for name in (
            f"optuna_{s}_{m}_latest.jsonpath",
            f"optuna_{s}_latest.jsonpath",
            "optuna_latest.jsonpath",
        ):
            with open(os.path.join(logs_dir, name), "w", encoding="utf-8") as f:
                f.write(abs_path + "\n")
    except OSError as exc:
        lg.warning("⚠️ jsonpath 사이드카 기록 실패: %s", exc)

    # 최종 JSON: 과적합·임계값 분포 진단 부착 후 브리핑
    note_l = (note or "").strip()
    if "최종" in note_l and abs_path and os.path.isfile(abs_path):
        try:
            import json as _json

            with open(abs_path, "r", encoding="utf-8") as f:
                _data = _json.load(f)
            attach_optuna_overfit_diagnostics(_data)
            with open(abs_path, "w", encoding="utf-8") as f:
                _json.dump(_data, f, indent=2, ensure_ascii=False)
            diag = _data.get("overfit_diagnostics") or {}
            lg.info(
                "📊 과적합위험 %s%% · 적용가능도 %s%% · 판정=%s",
                diag.get("overfit_risk_pct"),
                diag.get("apply_readiness_pct"),
                diag.get("verdict"),
            )
            print(
                f"OPTUNA_OVERFIT_RISK_PCT={diag.get('overfit_risk_pct')} "
                f"APPLY_READINESS_PCT={diag.get('apply_readiness_pct')} "
                f"VERDICT={diag.get('verdict')}",
                flush=True,
            )
        except Exception as exc:
            lg.warning("⚠️ overfit_diagnostics JSON 부착 실패: %s", exc)
        try:
            from kis_trader.backtest.optuna_briefing import write_briefing_for_json
            write_briefing_for_json(abs_path, log=lg)
        except Exception as exc:
            lg.warning("⚠️ Optuna 브리핑 실패: %s", exc)
    return abs_path