hoon/kis_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance Optuna integration and logging for backtesting framework

Browse Source 
Changes:
- Added new API endpoints for continuing and confirming Optuna jobs, allowing for better management of ongoing studies.
- Introduced detailed logging for tick feed tracking and order book processing, improving traceability of vendor performance during backtests.
- Updated database schema to include new fields for managing Optuna study results, enhancing the ability to track study progress and outcomes.
- Refactored existing functions to utilize the new logging and tracking features, ensuring consistency across the backtesting framework.

Impact:
- These enhancements improve the robustness and transparency of the Optuna backtesting process, facilitating better analysis and optimization of trading strategies.
This commit is contained in:
Your Name
2026-08-21 19:05:23 +09:00
parent 0ecac7cb95
commit 0780b2cdd0
76 changed files with 4648 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
kis_trader/backtest/optuna_postprocess_topn.py
View File

@@ -28,7 +28,8 @@ def _krw_int(v: Any) -> int:
return int(x)
def resolve_post_top_n(default: int = 5) -> int:
def resolve_post_top_n(default: int = 10) -> int:
"""후처리·웹 Top 표 행 수. 기본 10 (사후/학습/안정 공통)."""
return max(1, int(get_env_int("OPTUNA_POST_TOP_N", int(default))))
@@ -95,6 +96,7 @@ def _slim_ob(rec: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"ok": bool(rec.get("ok")),
"reason": rec.get("reason") or "",
"trade_count": int(rec.get("trade_count") or 0),
"fill_count": int(rec.get("fill_count") or 0),
"params": dict(rec.get("params") or {}),
"orig_stats": _slim_stats(rec.get("orig_stats")),
"recommended_stats": _slim_stats(rec.get("recommended_stats")),
@@ -337,6 +339,49 @@ def _median_params(params_list: List[Dict[str, Any]]) -> Dict[str, Any]:
return cons
def _ob_8way_attempt_note(pool: List[Dict[str, Any]]) -> Tuple[bool, str]:
"""8방 TPE를 돌렸는지(구JSON 아님)와 실패 사유.
Returns:
(attempted, why) — attempted=True 이면 재실행만으로 8방이 안 채워질 수 있음.
"""
bits: List[str] = []
saw_combos = False
for a in pool or []:
ob = a.get("orderbook") or {}
combos = ob.get("combos")
if isinstance(combos, dict) and combos:
saw_combos = True
r = str(ob.get("reason") or "").strip()
if r and r not in ("light_skip", "none"):
tc = ob.get("trade_count")
if r == "not_enough_trades" and tc is not None:
bit = "호가스냅%s건(<3)" % int(tc)
else:
bit = r
if bit not in bits:
bits.append(bit)
return (saw_combos or bool(bits), " · ".join(bits))
def ob_8way_web_hint(topn: Optional[Dict[str, Any]]) -> str:
"""웹 상태 한 줄. 8방이 채워졌으면 빈 문자열."""
anchors = list((topn or {}).get("postprocess_by_anchor") or [])
pool = [a for a in anchors if str(a.get("role") or "") in ("gated", "mode")]
if not pool:
return "8방 없음"
if any(
isinstance((a.get("orderbook") or {}).get("combos"), dict)
and (a.get("orderbook") or {}).get("combos")
for a in pool
):
return ""
attempted, why = _ob_8way_attempt_note(pool)
if attempted:
return why or "8방 미산출"
return "구JSON"
def _anchor_combos_map(ob: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
"""앵커 orderbook → combo_id dict. 구 JSON은 e/x/s 단독만 복원."""
if not isinstance(ob, dict):
@@ -400,8 +445,12 @@ def _axis_slice_from_combo(
def _consensus_from_anchors(anchors: List[Dict[str, Any]], strategy: str) -> Dict[str, Any]:
"""gated+mode. live·stable 제외. 호가=8방 중 median PnL 최고 방 + median 파라미터."""
pool = [a for a in anchors if str(a.get("role") or "") in ("gated", "mode")]
"""gated(+learn 폴백)+mode. live·stable 제외. 호가=8방 median PnL 최고 방."""
# 사후합격 없으면 학습 Top 후처리(learn)를 합의에 넣음 — WR/PF 때문에 호가방 학습이 비지 않게
roles = ("gated", "mode")
if not any(str(a.get("role") or "") == "gated" for a in (anchors or [])):
roles = ("gated", "learn", "mode")
pool = [a for a in anchors if str(a.get("role") or "") in roles]
by_combo: Dict[str, List[Dict[str, Any]]] = {cid: [] for cid in _COMBO_IDS}
for a in pool:
combos = _anchor_combos_map(a.get("orderbook") or {})
@@ -429,7 +478,11 @@ def _consensus_from_anchors(anchors: List[Dict[str, Any]], strategy: str) -> Dic
exit_c: Dict[str, Any] = {"ok": False, "params": {}, "n": 0}
stop_c: Dict[str, Any] = {"ok": False, "params": {}, "n": 0}
ob_merged: Dict[str, Any] = {}
note = "합의=Top5(+mode) 8방 중 median PnL 최고 방 + median 파라미터. 실매 참고행 제외."
note = (
"합의=Top5(+mode) 8방 중 median PnL 최고 방 + median 파라미터. 실매 참고행 제외."
if any(str(a.get("role") or "") == "gated" for a in (anchors or []))
else "합의=학습Top(+mode) — 사후합격(gated) 0건이라 WR/PF 미달 후보로 호가방 학습. 실매 참고행 제외."
)
if best_cid:
recs = by_combo[best_cid]
@@ -475,7 +528,15 @@ def _consensus_from_anchors(anchors: List[Dict[str, Any]], strategy: str) -> Dic
ob_merged = dict(entry.get("params") or {})
ob_merged.update(exit_c.get("params") or {})
ob_merged.update(stop_c.get("params") or {})
note = "합의=구JSON 축분리 median(8방 없음). 후처리 재실행 권장."
attempted, why = _ob_8way_attempt_note(pool)
if attempted:
note = (
"합의=8방 유효 방 없음(%s). 축분리 median 폴백. "
"호가스냅이 늘지 않으면 재실행해도 동일."
% (why or "TPE 실패")
)
else:
note = "합의=구JSON 축분리 median(8방 없음). 후처리 재실행 권장."
ws_params_list = [
dict((a.get("whipsaw") or {}).get("params") or {})
@@ -635,6 +696,86 @@ def _reuse_ob_ws(anchors: List[Dict[str, Any]], trial: Any) -> Optional[Dict[str
return None
def append_learn_postprocess_anchors(
data: Dict[str, Any],
anchors: List[Dict[str, Any]],
*,
strat: str,
strat_u: str,
top_n: int,
do_ob: bool,
evaluate_fn: Optional[EvalFn],
lg: logging.Logger,
ob_n_trials: int = 0,
) -> None:
"""
사후합격(gated)이 비었을 때 학습 TopN(results)에도 호가/휩쏘 후처리.
탐색은 WR/PF=0 이라 −PnL도 TPE에 들어가는데, 후처리만 gated(WR40/PF1)에
묶이면 gated=0 일 때 호가 8방이 mode/live만 남고 학습 후보가 통째 제외됨.
→ 후처리도 ‘끝난 뒤 후보 고르기’이지 ‘처음부터 제외’가 아니게 learn 폴백.
"""
if any(str(a.get("role") or "") == "gated" for a in anchors):
return
if any(str(a.get("role") or "") == "learn" for a in anchors):
return
learn = list(
(data or {}).get("results")
or (data or {}).get("results_all")
or []
)[: max(1, int(top_n or 5))]
if not learn:
return
lg.info(
"📌 [후처리] 사후합격 0건 → 학습 Top%d 에 호가/휩쏘 후처리 (WR/PF 사후게이트와 분리)",
len(learn),
)
for i, row in enumerate(learn, start=1):
if not isinstance(row, dict):
continue
combo = _combo_from_row(row)
fills: List[Dict[str, Any]] = []
if do_ob:
from kis_trader.backtest import optuna_post_progress as opp
opp.next_unit(f"learn#{i}", "학습Top 호가")
if do_ob and evaluate_fn is not None:
lg.info("📌 [후처리] learn#%d 백테 재실행 (호가/휩쏘 체결)", i)
fills = _replay_fills(evaluate_fn, combo, lg)
pnl = row.get("total_pnl")
trail = _slim_trail(_trail_for_pnl(strat, pnl))
ob = {"ok": False, "reason": "light_skip"}
ws = {"ok": False, "reason": "light_skip"}
if do_ob:
if fills:
ob = _ob_for_anchor(
strategy=strat_u, out_data=data, fills=fills, live=False,
lg=lg, n_trials=ob_n_trials,
)
ws = _ws_for_anchor(
strategy=strat_u, out_data=data, fills=fills, live=False, lg=lg,
)
elif evaluate_fn is None:
ob = {"ok": False, "reason": "no_replay_fills"}
ws = {"ok": False, "reason": "no_replay_fills"}
else:
ob = {"ok": False, "reason": "not_enough_trades", "trade_count": 0}
ws = {"ok": False, "reason": "not_enough_trades", "trade_count": 0}
anchors.append({
"id": f"learn#{i}",
"role": "learn",
"rank": i,
"optuna_trial_number": row.get("optuna_trial_number") or row.get("_trial_number"),
"total_pnl": _krw_int(pnl),
"total_trades": int(row.get("total_trades") or 0),
"win_rate": row.get("win_rate"),
"pf": row.get("pf"),
"orderbook": ob,
"whipsaw": ws,
"trail": trail,
"note": "학습 Top(사후게이트 미달 폴백) — 호가방 학습용 · DB적용은 gated 우선",
})
def append_stable_postprocess_anchors(
data: Dict[str, Any],
anchors: List[Dict[str, Any]],
@@ -652,7 +793,13 @@ def append_stable_postprocess_anchors(
return
if any(str(a.get("role") or "") == "stable" for a in anchors):
return
stable = list((data or {}).get("results_stable") or [])[: max(1, int(top_n or 5))]
from kis_trader.backtest.optuna_common import resolve_results_stable
stable, _meta = resolve_results_stable(data, top_n=max(1, int(top_n or 5)))
# 후처리 중 JSON에 비어 있으면 재구성분 반영 (다음 요약·앵커 일치)
if not list((data or {}).get("results_stable") or []) and stable:
data["results_stable"] = list(stable)
if _meta:
data["stable_gates"] = dict(_meta)
for i, row in enumerate(stable, start=1):
if not isinstance(row, dict):
continue
@@ -724,7 +871,7 @@ def ensure_stable_postprocess_on_payload(data: Dict[str, Any]) -> None:
before = len(anchors)
append_stable_postprocess_anchors(
data, anchors,
strat=strat, strat_u=strat_u, top_n=resolve_post_top_n(5),
strat=strat, strat_u=strat_u, top_n=resolve_post_top_n(10),
do_ob=False, evaluate_fn=None, lg=logger, ob_n_trials=0,
)
if len(anchors) != before:
@@ -754,7 +901,7 @@ def attach_topn_postprocess(
strat_u = "TAIL"
elif strat_u in ("SCALPING", "SCALP"):
strat_u = "SCALP"
top_n = resolve_post_top_n(5)
top_n = resolve_post_top_n(10)
do_ob = _run_ob_whipsaw_full() if run_ob_whipsaw is None else bool(run_ob_whipsaw)
gated = list(data.get("results_gated") or [])[:top_n]
@@ -762,14 +909,40 @@ def attach_topn_postprocess(
from kis_trader.backtest import optuna_post_progress as opp
stable_preview = list((data or {}).get("results_stable") or [])[: max(1, int(top_n or 5))] if _include_stable() else []
n_units = len(gated) + (len(stable_preview) if _include_stable() else 0)
stable_preview: List[Any] = []
if _include_stable():
from kis_trader.backtest.optuna_common import resolve_results_stable
stable_preview, _sg = resolve_results_stable(data, top_n=max(1, int(top_n or 5)))
if not list((data or {}).get("results_stable") or []) and stable_preview:
data["results_stable"] = list(stable_preview)
if _sg:
data["stable_gates"] = dict(_sg)
learn_preview: List[Any] = []
if not gated:
learn_preview = list(
(data or {}).get("results") or (data or {}).get("results_all") or []
)[: max(1, int(top_n or 5))]
n_units = len(gated) + len(learn_preview) + (len(stable_preview) if _include_stable() else 0)
if _include_mode():
n_units += 1
if _include_live():
n_units += 1
if do_ob:
opp.begin_job(lg, max(1, n_units))
try:
from kis_trader.backtest.optuna_feed_trace import (
log_bt_feed_chain_banner,
reset_bt_feed_sample_counter,
)
reset_bt_feed_sample_counter(postprocess=True)
log_bt_feed_chain_banner(context="Optuna후처리")
lg.info(
"🔎 [Optuna후처리] 호가 추천은 코어 TPE(필터OFF)와 별도 — "
"아래 [호가후처리]/[호가후처리샘플] 줄을 보면 됨"
)
except Exception:
pass
for i, row in enumerate(gated, start=1):
combo = _combo_from_row(row)
@@ -808,6 +981,13 @@ def attach_topn_postprocess(
"note": "사후합격 후보",
})
# gated 비면 학습 Top 에도 호가 후처리 (탐색 WR/PF=0 과 같은 정신)
append_learn_postprocess_anchors(
data, anchors,
strat=strat, strat_u=strat_u, top_n=top_n,
do_ob=do_ob, evaluate_fn=evaluate_fn, lg=lg, ob_n_trials=ob_n_trials,
)
append_stable_postprocess_anchors(
data, anchors,
strat=strat, strat_u=strat_u, top_n=top_n,