9ba9ab73b6
- Web UI: - Optuna 탭 추가 및 mode_combo (최빈값 조합), 사후합격 Top 10 시각화 기능 - 파라미터 분포(p25~p75, median, mode) 히스토그램 및 과적합(Overfit) 위험도 진단 UI 신설 - 체크박스 렌더링 깨짐 현상을 네이티브(appearance: auto)로 강제 복구 (CSS) - 다단 트레일링 스탑, 꼬리 진입/돌파 손절 등 고급 조건 설정 폼 UI 고도화 - Backend (Optuna Jobs): - CLI 환경에서 구동된 Optuna json 결과물을 웹 대시보드로 읽어오는 import 기능 강화 - JSON 메타데이터에 sort_by, mode, 호가 적용 여부 등 핵심 파라미터 파싱 누락 수정 - optuna_mode_combo.py 등 최빈값 조합 및 후보군 2차 검증을 위한 신규 모듈 추가 - DB & Execution: - WebSocket 호가/틱 피드 수집 통계(api_feed_collect_stats) 메모리 캐시 최적화 - KIS client 접속 키(approval_key) 등 인프라스트럭처 안정성 및 공유 관리 구조 개선 - 테스트 및 디버깅용 briefing 마크다운 자동 생성 기능 추가
687 lines
24 KiB
Python
687 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
kis_trader/backtest/optuna_mode_combo.py — Optuna Top-N 최빈 조합 추출·실측 백테
|
|
================================================================================
|
|
파람서치 종료 후 JSON/로그에 넣기 위한 공통 유틸.
|
|
|
|
기준:
|
|
1) results 중 total_pnl 있는 행만
|
|
2) PnL 내림차순 Top-N (기본 20, env OPTUNA_MODE_TOP_N)
|
|
3) 축별 단순 최빈(표수, PnL 가중 없음) → mode_combo
|
|
4) evaluate_fn(mode_combo) 로 1회 실측 백테 (게이트는 호출측 min_* 에 따름)
|
|
|
|
apply 는 하지 않음 — 확인용 리포트만.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections import Counter
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
from kis_trader.utils.env import get_env_float, get_env_from_db, get_env_int
|
|
from kis_trader.backtest.optuna_tpe_common import finalize_ratchet_combo
|
|
|
|
logger = logging.getLogger("optuna_mode_combo")
|
|
|
|
EvalFn = Callable[[Dict[str, Any]], Optional[Dict[str, Any]]]
|
|
|
|
|
|
def resolve_mode_top_n(default: int = 20) -> int:
|
|
"""Top-N — env OPTUNA_MODE_TOP_N (기본 20)."""
|
|
n = int(get_env_int("OPTUNA_MODE_TOP_N", int(default)))
|
|
return max(1, n)
|
|
|
|
|
|
def resolve_mode_pool_kind() -> str:
|
|
"""
|
|
mode_combo / mode Top10 / 2차 그리드 밴드 풀.
|
|
positive(기본)=PnL>0 전체 · gated=results_gated · top_n=Top-N PnL.
|
|
"""
|
|
raw = str(get_env_from_db("OPTUNA_MODE_POOL", "positive") or "positive").strip().lower()
|
|
if raw in ("positive", "gated", "top_n"):
|
|
return raw
|
|
return "positive"
|
|
|
|
|
|
def _valid_pnl_rows(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
return [
|
|
r for r in (results or [])
|
|
if r.get("total_pnl") is not None and abs(float(r.get("total_pnl") or 0)) < 1e15
|
|
]
|
|
|
|
|
|
def select_mode_pool_rows(
|
|
results: List[Dict[str, Any]],
|
|
*,
|
|
data: Optional[Dict[str, Any]] = None,
|
|
top_n: Optional[int] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
"""mode_combo·밴드·2차 narrow 공통 trial 풀."""
|
|
kind = resolve_mode_pool_kind()
|
|
n = int(top_n) if top_n is not None else resolve_mode_top_n(20)
|
|
rows = _valid_pnl_rows(results)
|
|
if kind == "gated" and data:
|
|
gated = [r for r in list(data.get("results_gated") or []) if isinstance(r, dict)]
|
|
gated = _valid_pnl_rows(gated)
|
|
if gated:
|
|
return gated
|
|
if kind == "positive":
|
|
pos = [r for r in rows if float(r.get("total_pnl") or 0) > 0]
|
|
if pos:
|
|
return pos
|
|
rows.sort(
|
|
key=lambda r: (
|
|
-float(r.get("total_pnl") or 0),
|
|
-float(r.get("win_rate") or 0),
|
|
-int(r.get("total_trades") or 0),
|
|
)
|
|
)
|
|
return rows[: max(1, n)]
|
|
rows.sort(
|
|
key=lambda r: (
|
|
-float(r.get("total_pnl") or 0),
|
|
-float(r.get("win_rate") or 0),
|
|
-int(r.get("total_trades") or 0),
|
|
)
|
|
)
|
|
return rows[: max(1, n)]
|
|
|
|
|
|
def mode_combo_from_results(
|
|
results: List[Dict[str, Any]],
|
|
*,
|
|
top_n: int = 20,
|
|
grid_keys: Optional[List[str]] = None,
|
|
params_key: str = "params",
|
|
data: Optional[Dict[str, Any]] = None,
|
|
pool_rows: Optional[List[Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
풀(PnL 양수 전체 등) 축별 최빈 → mode_combo + 빈도 메타.
|
|
|
|
Returns:
|
|
{
|
|
"top_n": int,
|
|
"pool_size": int,
|
|
"pool_kind": str,
|
|
"params": {축: 최빈값},
|
|
"freq": {축: {"value": ..., "count": n, "of": pool}},
|
|
"top_pnls": [...],
|
|
}
|
|
"""
|
|
pool_kind = resolve_mode_pool_kind()
|
|
pool = list(pool_rows) if pool_rows is not None else select_mode_pool_rows(
|
|
results, data=data, top_n=top_n,
|
|
)
|
|
if not pool:
|
|
return {
|
|
"top_n": int(top_n),
|
|
"pool_size": 0,
|
|
"pool_kind": pool_kind,
|
|
"params": {},
|
|
"freq": {},
|
|
"top_pnls": [],
|
|
}
|
|
|
|
# 축 집합: grid_keys 우선, 없으면 Top pool params 합집합
|
|
keys: List[str] = []
|
|
if grid_keys:
|
|
keys = [k for k in grid_keys if k]
|
|
if not keys:
|
|
seen = set()
|
|
for r in pool:
|
|
for k in (r.get(params_key) or {}).keys():
|
|
if k not in seen:
|
|
seen.add(k)
|
|
keys.append(k)
|
|
|
|
params: Dict[str, Any] = {}
|
|
freq: Dict[str, Any] = {}
|
|
for k in keys:
|
|
c: Counter = Counter()
|
|
samples: Dict[str, Any] = {}
|
|
for r in pool:
|
|
v = (r.get(params_key) or {}).get(k)
|
|
if v is None and params_key != "merged_params":
|
|
v = (r.get("merged_params") or {}).get(k)
|
|
s = str(v)
|
|
c[s] += 1
|
|
samples.setdefault(s, v)
|
|
if not c:
|
|
continue
|
|
best_s, cnt = c.most_common(1)[0]
|
|
params[k] = samples[best_s]
|
|
freq[k] = {"value": params[k], "count": int(cnt), "of": len(pool)}
|
|
|
|
return {
|
|
"top_n": int(top_n),
|
|
"pool_size": len(pool),
|
|
"pool_kind": pool_kind,
|
|
"params": params,
|
|
"freq": freq,
|
|
"top_pnls": [float(r.get("total_pnl") or 0) for r in pool[:10]],
|
|
}
|
|
|
|
|
|
def _bt_summary(result: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
"""실측 요약 — 사후합격/안정 Top 표와 같은 일별 안정 필드도 유지.
|
|
|
|
(예전엔 pnl·trades·wr·pf만 남겨 mode 표에 손실일·최악일·안정점수가 — 로 비었음)
|
|
"""
|
|
if not result:
|
|
return {
|
|
"ok": False,
|
|
"total_pnl": None,
|
|
"total_trades": None,
|
|
"win_rate": None,
|
|
"pf": None,
|
|
"note": "evaluate returned None (게이트·0건·invalid)",
|
|
}
|
|
out: Dict[str, Any] = {
|
|
"ok": True,
|
|
"total_pnl": float(result.get("total_pnl") or 0),
|
|
"total_trades": int(result.get("total_trades") or 0),
|
|
"win_rate": float(result.get("win_rate") or 0),
|
|
"pf": float(result.get("pf") or 0) if result.get("pf") is not None else None,
|
|
"score": float(result.get("score") or 0) if result.get("score") is not None else None,
|
|
}
|
|
# attach_daily_stability 가 evaluate_* 에 붙인 키 — 웹 mode 표 컬럼용
|
|
for k in (
|
|
"stability_score",
|
|
"n_losing_days",
|
|
"n_active_days",
|
|
"worst_day_pnl",
|
|
"best_day_pnl",
|
|
"daily_pnl_mean",
|
|
"daily_pnl_std",
|
|
"stability_lambda",
|
|
"daily_pnl",
|
|
):
|
|
if k in result and result.get(k) is not None:
|
|
out[k] = result.get(k)
|
|
return out
|
|
|
|
|
|
def _attach_best_trial_trades(
|
|
out_data: Dict[str, Any],
|
|
evaluate_fn: EvalFn,
|
|
*,
|
|
log: logging.Logger,
|
|
) -> None:
|
|
"""results[0](#1 best) 체결을 export 시 1회 재실측해 JSON에 남김 (정합 diff용)."""
|
|
from kis_trader.backtest.optuna_common import slim_trades_for_optuna_json
|
|
|
|
res0 = (out_data.get("results") or [None])[0]
|
|
if not isinstance(res0, dict):
|
|
return
|
|
combo = dict(res0.get("params") or {})
|
|
if not combo:
|
|
return
|
|
try:
|
|
raw = evaluate_fn(dict(combo))
|
|
except Exception as exc:
|
|
log.warning("⚠️ best 체결 재실측 예외: %s", exc)
|
|
return
|
|
if not isinstance(raw, dict):
|
|
log.warning("⚠️ best 체결 재실측 실패(게이트/None)")
|
|
return
|
|
fills = list(raw.pop("_trades", None) or [])
|
|
slim = slim_trades_for_optuna_json(fills)
|
|
res0["_trades"] = slim
|
|
out_data["best_trial_trades"] = slim
|
|
# trial 기록값과 export 시점 재실측이 다르면 바로 보이게
|
|
out_data["best_trial_reeval"] = {
|
|
"total_pnl": raw.get("total_pnl"),
|
|
"total_trades": raw.get("total_trades"),
|
|
"win_rate": raw.get("win_rate"),
|
|
"pf": raw.get("pf"),
|
|
"mdd": raw.get("mdd"),
|
|
"recorded_total_pnl": res0.get("total_pnl"),
|
|
"recorded_total_trades": res0.get("total_trades"),
|
|
"delta_pnl": (
|
|
float(raw.get("total_pnl") or 0) - float(res0.get("total_pnl") or 0)
|
|
),
|
|
"delta_trades": (
|
|
int(raw.get("total_trades") or 0) - int(res0.get("total_trades") or 0)
|
|
),
|
|
"note": (
|
|
"export 직후 동일 evaluate_fn 재실측. "
|
|
"delta≠0 이면 trial 기록과 엔진/데이터 드리프트."
|
|
),
|
|
}
|
|
log.info(
|
|
"🧾 [best 체결저장] n=%s | reeval_pnl=%s recorded_pnl=%s Δ=%+.0f",
|
|
len(slim),
|
|
raw.get("total_pnl"),
|
|
res0.get("total_pnl"),
|
|
float(raw.get("total_pnl") or 0) - float(res0.get("total_pnl") or 0),
|
|
)
|
|
|
|
|
|
def enrich_out_data_with_mode_combo(
|
|
out_data: Dict[str, Any],
|
|
*,
|
|
evaluate_fn: Optional[EvalFn] = None,
|
|
top_n: Optional[int] = None,
|
|
grid_keys: Optional[List[str]] = None,
|
|
params_key: str = "params",
|
|
log: Optional[logging.Logger] = None,
|
|
on_partial_save: Optional[Callable[[Dict[str, Any]], None]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
out_data['results'] 기준 최빈 추출 → (선택) 실측 백테 → out_data['mode_combo'] 기록 + 로그.
|
|
|
|
evaluate_fn: mode params → evaluate_*_param_combo 결과 dict 또는 None.
|
|
on_partial_save: 최빈 params 기록 직후(실측 전) 호출 — JSON에 mode_combo가 남도록.
|
|
"""
|
|
lg = log or logger
|
|
n = int(top_n) if top_n is not None else resolve_mode_top_n(20)
|
|
keys = grid_keys or list(out_data.get("grid_keys") or [])
|
|
mode_meta = mode_combo_from_results(
|
|
list(out_data.get("results") or []),
|
|
top_n=n,
|
|
grid_keys=keys or None,
|
|
params_key=params_key,
|
|
data=out_data,
|
|
)
|
|
# 래칫 숫자축 최빈 → 엔진용 ratchet_tiers 재조립 (불일치 방지)
|
|
strat = str(out_data.get("strategy") or "").strip().lower()
|
|
off_tok = "off" if strat == "tail" else ""
|
|
mode_params = finalize_ratchet_combo(
|
|
dict(mode_meta.get("params") or {}),
|
|
off_token=off_tok,
|
|
)
|
|
report: Dict[str, Any] = {
|
|
"method": "pool_per_axis_mode",
|
|
"top_n": mode_meta["top_n"],
|
|
"pool_size": mode_meta["pool_size"],
|
|
"pool_kind": mode_meta.get("pool_kind") or resolve_mode_pool_kind(),
|
|
"params": mode_params,
|
|
"freq": mode_meta["freq"],
|
|
"top_pnls": mode_meta["top_pnls"],
|
|
"backtest": None,
|
|
"vs_best": None,
|
|
"note": "trial 번호 없음(축별 최빈 조립). optuna_best_trial_number 와 별개.",
|
|
}
|
|
|
|
best_pnl = None
|
|
best_tr = None
|
|
res0 = (out_data.get("results") or [None])[0]
|
|
if res0:
|
|
best_pnl = float(res0.get("total_pnl") or 0)
|
|
best_tr = int(res0.get("total_trades") or 0)
|
|
|
|
lg.info(
|
|
"📊 [mode] pool(%s) 최빈 추출 | pool=%d | top_pnls=%s",
|
|
mode_meta.get("pool_kind") or resolve_mode_pool_kind(),
|
|
mode_meta["pool_size"],
|
|
mode_meta["top_pnls"][:5],
|
|
)
|
|
if mode_params:
|
|
# 축별 빈도 요약 (짧게)
|
|
bits = []
|
|
for k, meta in list(mode_meta["freq"].items())[:12]:
|
|
bits.append(f"{k}={meta['value']}({meta['count']}/{meta['of']})")
|
|
lg.info("📊 [mode] params(일부): %s", " | ".join(bits))
|
|
if "ratchet_tiers" in mode_params:
|
|
lg.info("📊 [mode] ratchet_tiers 재조립=%r", mode_params.get("ratchet_tiers"))
|
|
|
|
# 실측 전에 먼저 JSON에 박아 둠 (실측 중 죽어도 mode_combo.params 는 남음)
|
|
out_data["mode_combo"] = report
|
|
if on_partial_save is not None:
|
|
try:
|
|
on_partial_save(out_data)
|
|
except Exception as exc:
|
|
lg.warning("⚠️ mode_combo 부분저장 실패: %s", exc)
|
|
|
|
# #1 best 체결 — mode 실측 전에 저장 (후처리가 evaluate를 여러 번 돌려도 best는 1회)
|
|
if evaluate_fn is not None:
|
|
_attach_best_trial_trades(out_data, evaluate_fn, log=lg)
|
|
|
|
mode_fills: List[Dict[str, Any]] = []
|
|
if evaluate_fn is not None and mode_params:
|
|
try:
|
|
from kis_trader.backtest.optuna_common import slim_trades_for_optuna_json
|
|
|
|
bt = evaluate_fn(dict(mode_params))
|
|
if isinstance(bt, dict):
|
|
mode_fills = list(bt.pop("_trades", None) or [])
|
|
report["backtest"] = _bt_summary(bt)
|
|
# 후처리용 raw fills 는 mode_fills 로 유지 + JSON에는 슬림 저장
|
|
if report["backtest"].get("ok") and isinstance(report.get("backtest"), dict):
|
|
report["backtest"]["_trades"] = slim_trades_for_optuna_json(mode_fills)
|
|
if report["backtest"].get("ok"):
|
|
lg.info(
|
|
"🧪 [mode] 실측 백테 | pnl=%s | trades=%s | wr=%.1f%% | pf=%s | _trades=%s",
|
|
report["backtest"]["total_pnl"],
|
|
report["backtest"]["total_trades"],
|
|
float(report["backtest"]["win_rate"] or 0),
|
|
report["backtest"].get("pf"),
|
|
len(report["backtest"].get("_trades") or []),
|
|
)
|
|
else:
|
|
lg.warning("🧪 [mode] 실측 백테 실패/게이트: %s", report["backtest"].get("note"))
|
|
except Exception as exc:
|
|
report["backtest"] = {"ok": False, "error": str(exc)}
|
|
lg.warning("🧪 [mode] 실측 백테 예외: %s", exc)
|
|
mode_fills = []
|
|
|
|
if best_pnl is not None and report.get("backtest") and report["backtest"].get("ok"):
|
|
mode_pnl = float(report["backtest"]["total_pnl"] or 0)
|
|
best_wr = float(res0.get("win_rate") or 0) if res0 else None
|
|
best_pf = float(res0.get("pf") or 0) if res0 and res0.get("pf") is not None else None
|
|
mode_wr = report["backtest"].get("win_rate")
|
|
mode_pf = report["backtest"].get("pf")
|
|
report["vs_best"] = {
|
|
"best_pnl": best_pnl,
|
|
"best_trades": best_tr,
|
|
"best_wr": best_wr,
|
|
"best_pf": best_pf,
|
|
"mode_pnl": mode_pnl,
|
|
"mode_trades": report["backtest"].get("total_trades"),
|
|
"mode_wr": float(mode_wr) if mode_wr is not None else None,
|
|
"mode_pf": float(mode_pf) if mode_pf is not None else None,
|
|
"delta_pnl": round(mode_pnl - best_pnl, 2),
|
|
"delta_wr": (
|
|
round(float(mode_wr) - float(best_wr), 2)
|
|
if mode_wr is not None and best_wr is not None
|
|
else None
|
|
),
|
|
"delta_pf": (
|
|
round(float(mode_pf) - float(best_pf), 2)
|
|
if mode_pf is not None and best_pf is not None
|
|
else None
|
|
),
|
|
}
|
|
lg.info(
|
|
"📐 [mode vs #1] best_pnl=%s (%s건 wr=%.1f%% pf=%s) | "
|
|
"mode_pnl=%s (%s건 wr=%.1f%% pf=%s) | Δpnl=%+.0f Δwr=%+.1f",
|
|
best_pnl,
|
|
best_tr,
|
|
float(best_wr or 0),
|
|
best_pf,
|
|
mode_pnl,
|
|
report["backtest"].get("total_trades"),
|
|
float(mode_wr or 0),
|
|
mode_pf,
|
|
mode_pnl - best_pnl,
|
|
(float(mode_wr) - float(best_wr)) if mode_wr is not None and best_wr is not None else 0.0,
|
|
)
|
|
|
|
out_data["mode_combo"] = report
|
|
# TopN 후처리(호가/휩쏘/트레일) — 실매 엔진 미변경. 합의값이 기존 단일 recommend 키를 대체.
|
|
try:
|
|
from kis_trader.backtest.optuna_postprocess_topn import attach_topn_postprocess
|
|
|
|
attach_topn_postprocess(
|
|
out_data,
|
|
evaluate_fn=evaluate_fn,
|
|
mode_fills=mode_fills,
|
|
log=lg,
|
|
run_ob_whipsaw=None, # TPE 호가축 ON 이면 사후 8방 기본 OFF (_run_ob_whipsaw_full)
|
|
)
|
|
except Exception as exc:
|
|
lg.warning("⚠️ TopN 후처리 첨부 실패: %s", exc)
|
|
try:
|
|
from kis_trader.backtest.optuna_daily_trail_recommend import (
|
|
attach_daily_trail_recommend,
|
|
)
|
|
attach_daily_trail_recommend(out_data, log=lg)
|
|
except Exception as exc2:
|
|
lg.warning("⚠️ daily_trail_recommend 폴백 실패: %s", exc2)
|
|
|
|
# mode Top10 — 웹 표·apply source=consensus (구 JSON은 웹에서 재계산)
|
|
try:
|
|
from kis_trader.backtest.optuna_postprocess_topn import resolve_post_top_n
|
|
|
|
ui_n = resolve_post_top_n(10)
|
|
mode_rows, mode_meta = build_results_mode_consensus_tier(
|
|
list(out_data.get("results_all") or out_data.get("results") or []),
|
|
top_n=ui_n,
|
|
grid_keys=keys or None,
|
|
params_key=params_key,
|
|
data=out_data,
|
|
)
|
|
out_data["results_mode"] = mode_rows
|
|
out_data["mode_consensus_meta"] = mode_meta
|
|
except Exception as exc:
|
|
lg.warning("⚠️ results_mode(mode Top10) 첨부 실패: %s", exc)
|
|
|
|
return out_data
|
|
|
|
|
|
def _percentile_sorted(sorted_vals: List[float], p: float) -> float:
|
|
if not sorted_vals:
|
|
return 0.0
|
|
if len(sorted_vals) == 1:
|
|
return sorted_vals[0]
|
|
idx = (len(sorted_vals) - 1) * p
|
|
lo = int(idx)
|
|
hi = min(lo + 1, len(sorted_vals) - 1)
|
|
w = idx - lo
|
|
return sorted_vals[lo] * (1.0 - w) + sorted_vals[hi] * w
|
|
|
|
|
|
def _row_param_value(row: Dict[str, Any], key: str, *, params_key: str = "params") -> Any:
|
|
params = row.get(params_key) or row.get("merged_params") or {}
|
|
if not isinstance(params, dict):
|
|
params = {}
|
|
v = params.get(key)
|
|
if v is None and params_key != "merged_params":
|
|
v = (row.get("merged_params") or {}).get(key)
|
|
return v
|
|
|
|
|
|
def _coerce_numeric(v: Any) -> Optional[float]:
|
|
if isinstance(v, bool):
|
|
return 1.0 if v else 0.0
|
|
try:
|
|
return float(v)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _build_mode_band_profile(
|
|
pool: List[Dict[str, Any]],
|
|
keys: List[str],
|
|
*,
|
|
params_key: str = "params",
|
|
) -> Dict[str, Dict[str, Any]]:
|
|
"""
|
|
Top pool 각 축 — 숫자면 p25~p75 밴드(흔한 구간), 아니면 categorical mode.
|
|
"""
|
|
profile: Dict[str, Dict[str, Any]] = {}
|
|
for k in keys:
|
|
raw_vals: List[Any] = []
|
|
for row in pool:
|
|
v = _row_param_value(row, k, params_key=params_key)
|
|
if v is not None:
|
|
raw_vals.append(v)
|
|
if not raw_vals:
|
|
continue
|
|
nums: List[float] = []
|
|
all_numeric = True
|
|
for v in raw_vals:
|
|
n = _coerce_numeric(v)
|
|
if n is None:
|
|
all_numeric = False
|
|
break
|
|
nums.append(n)
|
|
if all_numeric and nums:
|
|
s = sorted(nums)
|
|
p25 = _percentile_sorted(s, 0.25)
|
|
p50 = _percentile_sorted(s, 0.50)
|
|
p75 = _percentile_sorted(s, 0.75)
|
|
iqr = max(p75 - p25, abs(p50) * 0.05, 1e-9)
|
|
profile[k] = {
|
|
"kind": "numeric",
|
|
"p25": p25,
|
|
"p50": p50,
|
|
"p75": p75,
|
|
"iqr": iqr,
|
|
}
|
|
else:
|
|
c: Counter = Counter(str(v) for v in raw_vals)
|
|
mode_s, _cnt = c.most_common(1)[0]
|
|
sample = next(v for v in raw_vals if str(v) == mode_s)
|
|
profile[k] = {"kind": "categorical", "mode": sample}
|
|
return profile
|
|
|
|
|
|
def _trial_band_proximity(
|
|
row: Dict[str, Any],
|
|
profile: Dict[str, Dict[str, Any]],
|
|
*,
|
|
params_key: str = "params",
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
trial ↔ pool 흔한 구간(p25~p75) 근접도. 100%=모든 축이 밴드 안 또는 매우 가까움.
|
|
(구: 축값 완전 일치 개수 — tp 22% vs 4% 뒤섞임 원인)
|
|
"""
|
|
decay_iqr = max(0.1, float(get_env_float("OPTUNA_MODE_BAND_DECAY_IQR", 1.5)))
|
|
scores: List[float] = []
|
|
in_band = 0
|
|
total = 0
|
|
err_sum = 0.0
|
|
for k, band in profile.items():
|
|
rv = _row_param_value(row, k, params_key=params_key)
|
|
if rv is None:
|
|
continue
|
|
total += 1
|
|
if band.get("kind") == "numeric":
|
|
nv = _coerce_numeric(rv)
|
|
if nv is None:
|
|
mode_v = band.get("mode")
|
|
if mode_v is not None:
|
|
axis_s = 1.0 if str(rv) == str(mode_v) else 0.0
|
|
else:
|
|
axis_s = 0.0
|
|
if axis_s >= 1.0:
|
|
in_band += 1
|
|
else:
|
|
err_sum += 1.0
|
|
scores.append(axis_s)
|
|
continue
|
|
p25 = float(band["p25"])
|
|
p50 = float(band["p50"])
|
|
p75 = float(band["p75"])
|
|
iqr = float(band["iqr"])
|
|
if p25 <= nv <= p75:
|
|
axis_s = 1.0
|
|
in_band += 1
|
|
err_sum += 0.0
|
|
else:
|
|
dist = (p25 - nv) if nv < p25 else (nv - p75)
|
|
axis_s = max(0.0, 1.0 - dist / (iqr * decay_iqr))
|
|
err_sum += dist / iqr
|
|
else:
|
|
mode_v = band.get("mode")
|
|
axis_s = 1.0 if str(rv) == str(mode_v) else 0.0
|
|
if axis_s >= 1.0:
|
|
in_band += 1
|
|
err_sum += 0.0 if axis_s >= 1.0 else 1.0
|
|
scores.append(axis_s)
|
|
pct = (sum(scores) / float(len(scores)) * 100.0) if scores else 0.0
|
|
mean_err = (err_sum / float(total)) if total else 0.0
|
|
return {
|
|
"matched": in_band,
|
|
"total": total,
|
|
"pct": round(pct, 1),
|
|
"mean_band_err": round(mean_err, 4),
|
|
}
|
|
|
|
|
|
def _trial_consensus_match(
|
|
row: Dict[str, Any],
|
|
mode_params: Dict[str, Any],
|
|
*,
|
|
params_key: str = "params",
|
|
) -> Dict[str, Any]:
|
|
"""trial params 가 mode(축별 최빈) 와 몇 축 일치하는지."""
|
|
params = row.get(params_key) or row.get("merged_params") or {}
|
|
if not isinstance(params, dict):
|
|
params = {}
|
|
matched = 0
|
|
total = 0
|
|
for k, mv in (mode_params or {}).items():
|
|
rv = params.get(k)
|
|
if rv is None and params_key != "merged_params":
|
|
rv = (row.get("merged_params") or {}).get(k)
|
|
if rv is None:
|
|
continue
|
|
total += 1
|
|
if str(rv) == str(mv):
|
|
matched += 1
|
|
pct = (float(matched) / float(total) * 100.0) if total else 0.0
|
|
return {"matched": matched, "total": total, "pct": round(pct, 1)}
|
|
|
|
|
|
def build_results_mode_consensus_tier(
|
|
results: List[Dict[str, Any]],
|
|
*,
|
|
top_n: int = 10,
|
|
mode_top_n: Optional[int] = None,
|
|
grid_keys: Optional[List[str]] = None,
|
|
params_key: str = "params",
|
|
data: Optional[Dict[str, Any]] = None,
|
|
) -> tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
|
"""
|
|
mode Top10 — 풀(PnL 양수 전체 등) p25~p75 밴드에 **가장 가까운** trial 순.
|
|
|
|
밴드는 pool 전체에서 계산 · 후보는 pool 전 trial(양수 전체)에서 근접도 순.
|
|
mode_combo(1회 실측·trial 없음)와 달리 **실제 trial 번호**가 있음.
|
|
"""
|
|
mode_n = int(mode_top_n) if mode_top_n is not None else resolve_mode_top_n(20)
|
|
pool_kind = resolve_mode_pool_kind()
|
|
learn_pool = select_mode_pool_rows(list(results or []), data=data, top_n=mode_n)
|
|
mode_meta = mode_combo_from_results(
|
|
list(results or []),
|
|
top_n=mode_n,
|
|
grid_keys=grid_keys,
|
|
params_key=params_key,
|
|
pool_rows=learn_pool,
|
|
)
|
|
meta: Dict[str, Any] = {
|
|
"mode_top_n": mode_n,
|
|
"mode_pool_size": int(mode_meta.get("pool_size") or 0),
|
|
"mode_pool_kind": pool_kind,
|
|
"mode_params_keys": len(mode_meta.get("params") or {}),
|
|
"scoring": "band_proximity_p25_p75",
|
|
"note": f"pool={pool_kind} · 축별 p25~p75 밴드 근접도(오차↓) · OPTUNA_MODE_POOL",
|
|
}
|
|
if not mode_meta.get("pool_size"):
|
|
meta["note"] = "mode pool 없음 — results·grid_keys 확인"
|
|
return [], meta
|
|
|
|
candidate_pool = list(learn_pool)
|
|
keys: List[str] = []
|
|
if grid_keys:
|
|
keys = [k for k in grid_keys if k]
|
|
if not keys:
|
|
seen: set = set()
|
|
for r in learn_pool:
|
|
for k in (r.get(params_key) or {}).keys():
|
|
if k not in seen:
|
|
seen.add(k)
|
|
keys.append(k)
|
|
profile = _build_mode_band_profile(learn_pool, keys, params_key=params_key)
|
|
meta["band_axes"] = len(profile)
|
|
scored: List[Dict[str, Any]] = []
|
|
for r in candidate_pool:
|
|
m = _trial_band_proximity(r, profile, params_key=params_key)
|
|
row = dict(r)
|
|
row["consensus_match_pct"] = m["pct"]
|
|
row["consensus_match_n"] = m["matched"]
|
|
row["consensus_match_of"] = m["total"]
|
|
row["consensus_band_err"] = m.get("mean_band_err")
|
|
scored.append(row)
|
|
scored.sort(
|
|
key=lambda r: (
|
|
-float(r.get("consensus_match_pct") or 0),
|
|
float(r.get("consensus_band_err") or 999.0),
|
|
-float(r.get("total_pnl") or 0),
|
|
-float(r.get("score") or 0),
|
|
)
|
|
)
|
|
return scored[: max(1, int(top_n))], meta
|