hoon/kis_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backtest): 대대적인 Optuna 백테스트 웹 UI 및 백엔드 파이프라인 개편

Browse Source 
- Web UI:
  - Optuna 탭 추가 및 mode_combo (최빈값 조합), 사후합격 Top 10 시각화 기능
  - 파라미터 분포(p25~p75, median, mode) 히스토그램 및 과적합(Overfit) 위험도 진단 UI 신설
  - 체크박스 렌더링 깨짐 현상을 네이티브(appearance: auto)로 강제 복구 (CSS)
  - 다단 트레일링 스탑, 꼬리 진입/돌파 손절 등 고급 조건 설정 폼 UI 고도화

- Backend (Optuna Jobs):
  - CLI 환경에서 구동된 Optuna json 결과물을 웹 대시보드로 읽어오는 import 기능 강화
  - JSON 메타데이터에 sort_by, mode, 호가 적용 여부 등 핵심 파라미터 파싱 누락 수정
  - optuna_mode_combo.py 등 최빈값 조합 및 후보군 2차 검증을 위한 신규 모듈 추가

- DB & Execution:
  - WebSocket 호가/틱 피드 수집 통계(api_feed_collect_stats) 메모리 캐시 최적화
  - KIS client 접속 키(approval_key) 등 인프라스트럭처 안정성 및 공유 관리 구조 개선
  - 테스트 및 디버깅용 briefing 마크다운 자동 생성 기능 추가
This commit is contained in:
Your Name
2026-09-01 02:47:51 +09:00
parent 3d0255519e
commit 9ba9ab73b6
92 changed files with 8211 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

353
kis_trader/backtest/optuna_mode_combo.py
View File

@@ -18,7 +18,7 @@ import logging
from collections import Counter
from typing import Any, Callable, Dict, List, Optional
from kis_trader.utils.env import get_env_int
from kis_trader.utils.env import get_env_float, get_env_from_db, get_env_int
from kis_trader.backtest.optuna_tpe_common import finalize_ratchet_combo
logger = logging.getLogger("optuna_mode_combo")
@@ -32,29 +32,51 @@ def resolve_mode_top_n(default: int = 20) -> int:
return max(1, n)
def mode_combo_from_results(
results: List[Dict[str, Any]],
*,
top_n: int = 20,
grid_keys: Optional[List[str]] = None,
params_key: str = "params",
) -> Dict[str, Any]:
def resolve_mode_pool_kind() -> str:
"""
Top-N(PnL) 축별 최빈 → mode_combo + 빈도 메타.
mode_combo / mode Top10 / 2차 그리드 밴드 풀.
positive(기본)=PnL>0 전체 · gated=results_gated · top_n=Top-N PnL.
"""
raw = str(get_env_from_db("OPTUNA_MODE_POOL", "positive") or "positive").strip().lower()
if raw in ("positive", "gated", "top_n"):
return raw
return "positive"
Returns:
{
"top_n": int,
"pool_size": int,
"params": {축: 최빈값},
"freq": {축: {"value": ..., "count": n, "of": pool}},
"top_pnls": [...],
}
"""
rows = [
def _valid_pnl_rows(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
return [
r for r in (results or [])
if r.get("total_pnl") is not None and abs(float(r.get("total_pnl") or 0)) < 1e15
]
def select_mode_pool_rows(
results: List[Dict[str, Any]],
*,
data: Optional[Dict[str, Any]] = None,
top_n: Optional[int] = None,
) -> List[Dict[str, Any]]:
"""mode_combo·밴드·2차 narrow 공통 trial 풀."""
kind = resolve_mode_pool_kind()
n = int(top_n) if top_n is not None else resolve_mode_top_n(20)
rows = _valid_pnl_rows(results)
if kind == "gated" and data:
gated = [r for r in list(data.get("results_gated") or []) if isinstance(r, dict)]
gated = _valid_pnl_rows(gated)
if gated:
return gated
if kind == "positive":
pos = [r for r in rows if float(r.get("total_pnl") or 0) > 0]
if pos:
return pos
rows.sort(
key=lambda r: (
-float(r.get("total_pnl") or 0),
-float(r.get("win_rate") or 0),
-int(r.get("total_trades") or 0),
)
)
return rows[: max(1, n)]
rows.sort(
key=lambda r: (
-float(r.get("total_pnl") or 0),
@@ -62,11 +84,40 @@ def mode_combo_from_results(
-int(r.get("total_trades") or 0),
)
)
pool = rows[: max(1, int(top_n))]
return rows[: max(1, n)]
def mode_combo_from_results(
results: List[Dict[str, Any]],
*,
top_n: int = 20,
grid_keys: Optional[List[str]] = None,
params_key: str = "params",
data: Optional[Dict[str, Any]] = None,
pool_rows: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""
풀(PnL 양수 전체 등) 축별 최빈 → mode_combo + 빈도 메타.
Returns:
{
"top_n": int,
"pool_size": int,
"pool_kind": str,
"params": {축: 최빈값},
"freq": {축: {"value": ..., "count": n, "of": pool}},
"top_pnls": [...],
}
"""
pool_kind = resolve_mode_pool_kind()
pool = list(pool_rows) if pool_rows is not None else select_mode_pool_rows(
results, data=data, top_n=top_n,
)
if not pool:
return {
"top_n": int(top_n),
"pool_size": 0,
"pool_kind": pool_kind,
"params": {},
"freq": {},
"top_pnls": [],
@@ -105,6 +156,7 @@ def mode_combo_from_results(
return {
"top_n": int(top_n),
"pool_size": len(pool),
"pool_kind": pool_kind,
"params": params,
"freq": freq,
"top_pnls": [float(r.get("total_pnl") or 0) for r in pool[:10]],
@@ -230,6 +282,7 @@ def enrich_out_data_with_mode_combo(
top_n=n,
grid_keys=keys or None,
params_key=params_key,
data=out_data,
)
# 래칫 숫자축 최빈 → 엔진용 ratchet_tiers 재조립 (불일치 방지)
strat = str(out_data.get("strategy") or "").strip().lower()
@@ -239,9 +292,10 @@ def enrich_out_data_with_mode_combo(
off_token=off_tok,
)
report: Dict[str, Any] = {
"method": "top_n_per_axis_mode",
"method": "pool_per_axis_mode",
"top_n": mode_meta["top_n"],
"pool_size": mode_meta["pool_size"],
"pool_kind": mode_meta.get("pool_kind") or resolve_mode_pool_kind(),
"params": mode_params,
"freq": mode_meta["freq"],
"top_pnls": mode_meta["top_pnls"],
@@ -258,8 +312,8 @@ def enrich_out_data_with_mode_combo(
best_tr = int(res0.get("total_trades") or 0)
lg.info(
"📊 [mode] Top-%d 최빈 추출 | pool=%d | top_pnls=%s",
mode_meta["top_n"],
"📊 [mode] pool(%s) 최빈 추출 | pool=%d | top_pnls=%s",
mode_meta.get("pool_kind") or resolve_mode_pool_kind(),
mode_meta["pool_size"],
mode_meta["top_pnls"][:5],
)
@@ -376,4 +430,257 @@ def enrich_out_data_with_mode_combo(
except Exception as exc2:
lg.warning("⚠️ daily_trail_recommend 폴백 실패: %s", exc2)
# mode Top10 — 웹 표·apply source=consensus (구 JSON은 웹에서 재계산)
try:
from kis_trader.backtest.optuna_postprocess_topn import resolve_post_top_n
ui_n = resolve_post_top_n(10)
mode_rows, mode_meta = build_results_mode_consensus_tier(
list(out_data.get("results_all") or out_data.get("results") or []),
top_n=ui_n,
grid_keys=keys or None,
params_key=params_key,
data=out_data,
)
out_data["results_mode"] = mode_rows
out_data["mode_consensus_meta"] = mode_meta
except Exception as exc:
lg.warning("⚠️ results_mode(mode Top10) 첨부 실패: %s", exc)
return out_data
def _percentile_sorted(sorted_vals: List[float], p: float) -> float:
if not sorted_vals:
return 0.0
if len(sorted_vals) == 1:
return sorted_vals[0]
idx = (len(sorted_vals) - 1) * p
lo = int(idx)
hi = min(lo + 1, len(sorted_vals) - 1)
w = idx - lo
return sorted_vals[lo] * (1.0 - w) + sorted_vals[hi] * w
def _row_param_value(row: Dict[str, Any], key: str, *, params_key: str = "params") -> Any:
params = row.get(params_key) or row.get("merged_params") or {}
if not isinstance(params, dict):
params = {}
v = params.get(key)
if v is None and params_key != "merged_params":
v = (row.get("merged_params") or {}).get(key)
return v
def _coerce_numeric(v: Any) -> Optional[float]:
if isinstance(v, bool):
return 1.0 if v else 0.0
try:
return float(v)
except (TypeError, ValueError):
return None
def _build_mode_band_profile(
pool: List[Dict[str, Any]],
keys: List[str],
*,
params_key: str = "params",
) -> Dict[str, Dict[str, Any]]:
"""
Top pool 각 축 — 숫자면 p25~p75 밴드(흔한 구간), 아니면 categorical mode.
"""
profile: Dict[str, Dict[str, Any]] = {}
for k in keys:
raw_vals: List[Any] = []
for row in pool:
v = _row_param_value(row, k, params_key=params_key)
if v is not None:
raw_vals.append(v)
if not raw_vals:
continue
nums: List[float] = []
all_numeric = True
for v in raw_vals:
n = _coerce_numeric(v)
if n is None:
all_numeric = False
break
nums.append(n)
if all_numeric and nums:
s = sorted(nums)
p25 = _percentile_sorted(s, 0.25)
p50 = _percentile_sorted(s, 0.50)
p75 = _percentile_sorted(s, 0.75)
iqr = max(p75 - p25, abs(p50) * 0.05, 1e-9)
profile[k] = {
"kind": "numeric",
"p25": p25,
"p50": p50,
"p75": p75,
"iqr": iqr,
}
else:
c: Counter = Counter(str(v) for v in raw_vals)
mode_s, _cnt = c.most_common(1)[0]
sample = next(v for v in raw_vals if str(v) == mode_s)
profile[k] = {"kind": "categorical", "mode": sample}
return profile
def _trial_band_proximity(
row: Dict[str, Any],
profile: Dict[str, Dict[str, Any]],
*,
params_key: str = "params",
) -> Dict[str, Any]:
"""
trial ↔ pool 흔한 구간(p25~p75) 근접도. 100%=모든 축이 밴드 안 또는 매우 가까움.
(구: 축값 완전 일치 개수 — tp 22% vs 4% 뒤섞임 원인)
"""
decay_iqr = max(0.1, float(get_env_float("OPTUNA_MODE_BAND_DECAY_IQR", 1.5)))
scores: List[float] = []
in_band = 0
total = 0
err_sum = 0.0
for k, band in profile.items():
rv = _row_param_value(row, k, params_key=params_key)
if rv is None:
continue
total += 1
if band.get("kind") == "numeric":
nv = _coerce_numeric(rv)
if nv is None:
mode_v = band.get("mode")
if mode_v is not None:
axis_s = 1.0 if str(rv) == str(mode_v) else 0.0
else:
axis_s = 0.0
if axis_s >= 1.0:
in_band += 1
else:
err_sum += 1.0
scores.append(axis_s)
continue
p25 = float(band["p25"])
p50 = float(band["p50"])
p75 = float(band["p75"])
iqr = float(band["iqr"])
if p25 <= nv <= p75:
axis_s = 1.0
in_band += 1
err_sum += 0.0
else:
dist = (p25 - nv) if nv < p25 else (nv - p75)
axis_s = max(0.0, 1.0 - dist / (iqr * decay_iqr))
err_sum += dist / iqr
else:
mode_v = band.get("mode")
axis_s = 1.0 if str(rv) == str(mode_v) else 0.0
if axis_s >= 1.0:
in_band += 1
err_sum += 0.0 if axis_s >= 1.0 else 1.0
scores.append(axis_s)
pct = (sum(scores) / float(len(scores)) * 100.0) if scores else 0.0
mean_err = (err_sum / float(total)) if total else 0.0
return {
"matched": in_band,
"total": total,
"pct": round(pct, 1),
"mean_band_err": round(mean_err, 4),
}
def _trial_consensus_match(
row: Dict[str, Any],
mode_params: Dict[str, Any],
*,
params_key: str = "params",
) -> Dict[str, Any]:
"""trial params 가 mode(축별 최빈) 와 몇 축 일치하는지."""
params = row.get(params_key) or row.get("merged_params") or {}
if not isinstance(params, dict):
params = {}
matched = 0
total = 0
for k, mv in (mode_params or {}).items():
rv = params.get(k)
if rv is None and params_key != "merged_params":
rv = (row.get("merged_params") or {}).get(k)
if rv is None:
continue
total += 1
if str(rv) == str(mv):
matched += 1
pct = (float(matched) / float(total) * 100.0) if total else 0.0
return {"matched": matched, "total": total, "pct": round(pct, 1)}
def build_results_mode_consensus_tier(
results: List[Dict[str, Any]],
*,
top_n: int = 10,
mode_top_n: Optional[int] = None,
grid_keys: Optional[List[str]] = None,
params_key: str = "params",
data: Optional[Dict[str, Any]] = None,
) -> tuple[List[Dict[str, Any]], Dict[str, Any]]:
"""
mode Top10 — 풀(PnL 양수 전체 등) p25~p75 밴드에 **가장 가까운** trial 순.
밴드는 pool 전체에서 계산 · 후보는 pool 전 trial(양수 전체)에서 근접도 순.
mode_combo(1회 실측·trial 없음)와 달리 **실제 trial 번호**가 있음.
"""
mode_n = int(mode_top_n) if mode_top_n is not None else resolve_mode_top_n(20)
pool_kind = resolve_mode_pool_kind()
learn_pool = select_mode_pool_rows(list(results or []), data=data, top_n=mode_n)
mode_meta = mode_combo_from_results(
list(results or []),
top_n=mode_n,
grid_keys=grid_keys,
params_key=params_key,
pool_rows=learn_pool,
)
meta: Dict[str, Any] = {
"mode_top_n": mode_n,
"mode_pool_size": int(mode_meta.get("pool_size") or 0),
"mode_pool_kind": pool_kind,
"mode_params_keys": len(mode_meta.get("params") or {}),
"scoring": "band_proximity_p25_p75",
"note": f"pool={pool_kind} · 축별 p25~p75 밴드 근접도(오차↓) · OPTUNA_MODE_POOL",
}
if not mode_meta.get("pool_size"):
meta["note"] = "mode pool 없음 — results·grid_keys 확인"
return [], meta
candidate_pool = list(learn_pool)
keys: List[str] = []
if grid_keys:
keys = [k for k in grid_keys if k]
if not keys:
seen: set = set()
for r in learn_pool:
for k in (r.get(params_key) or {}).keys():
if k not in seen:
seen.add(k)
keys.append(k)
profile = _build_mode_band_profile(learn_pool, keys, params_key=params_key)
meta["band_axes"] = len(profile)
scored: List[Dict[str, Any]] = []
for r in candidate_pool:
m = _trial_band_proximity(r, profile, params_key=params_key)
row = dict(r)
row["consensus_match_pct"] = m["pct"]
row["consensus_match_n"] = m["matched"]
row["consensus_match_of"] = m["total"]
row["consensus_band_err"] = m.get("mean_band_err")
scored.append(row)
scored.sort(
key=lambda r: (
-float(r.get("consensus_match_pct") or 0),
float(r.get("consensus_band_err") or 999.0),
-float(r.get("total_pnl") or 0),
-float(r.get("score") or 0),
)
)
return scored[: max(1, int(top_n))], meta