refactor: enhance Optuna backtesting framework, optimize orderbook filtering, and update database management utilities.
This commit is contained in:
Your Name
413
kis_trader/backtest/trade_orderbook_enrich.py
Normal file
413
kis_trader/backtest/trade_orderbook_enrich.py
Normal file
@@ -0,0 +1,413 @@
|
||||
"""
|
||||
거래내역 UI용 — 매수/매도 시각 근처 호가 스냅 부착.
|
||||
|
||||
필터 ON/OFF 와 무관: 수집된 ws_orderbook / ls_ws_orderbook 으로
|
||||
진입(매수) · 청산(매도) 시점 유동성을 각각 보여 준다.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
logger = logging.getLogger("trade_orderbook_enrich")
|
||||
|
||||
# 표시용 매칭 창 — 체결≠호가저장 이므로 실매 UI는 ±15분까지 근처 기록 허용
|
||||
_STRICT_DELTA_SEC = 180 # 판정(filter_eval) 정밀 창
|
||||
_SOFT_DELTA_SEC = 900 # 본체·표시용 완화 창 (±15분)
|
||||
_BODY_PREF_DELTA_SEC = 120 # 본체 우선 창
|
||||
_EXIT_OB_OR_DEFAULT = 0.4 # L3 OR 임계 표시용 (MOMENTUM_EXIT_OB_RATIO_MIN 기본)
|
||||
|
||||
|
||||
def _strategy_canon(strategy: str) -> str:
|
||||
s = (strategy or "").strip().upper()
|
||||
if s in ("SHORT", "TAIL_CATCH", "TAIL"):
|
||||
return "TAIL"
|
||||
if s in ("BO", "BREAKOUT"):
|
||||
return "BREAKOUT"
|
||||
if s in ("MOM", "MOMENTUM"):
|
||||
return "MOMENTUM"
|
||||
if s in ("SCALP", "SCALPING", "REVERSAL"):
|
||||
return "SCALP"
|
||||
if s.startswith("US_"):
|
||||
return "US"
|
||||
return s
|
||||
|
||||
|
||||
def _ts14(raw: Any) -> str:
|
||||
if raw is None:
|
||||
return ""
|
||||
s = str(raw).strip()
|
||||
if not s:
|
||||
return ""
|
||||
s = (
|
||||
s.replace("-", "")
|
||||
.replace(":", "")
|
||||
.replace(" ", "")
|
||||
.replace("T", "")
|
||||
.replace(".", "")
|
||||
)
|
||||
if len(s) < 8:
|
||||
return ""
|
||||
return (s + "000000")[:14]
|
||||
|
||||
|
||||
def _parse14(ts14: str) -> Optional[datetime]:
|
||||
st = (ts14 or "").strip()
|
||||
if len(st) < 12:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(st[:14], "%Y%m%d%H%M%S")
|
||||
except ValueError:
|
||||
try:
|
||||
return datetime.strptime(st[:12], "%Y%m%d%H%M")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _entry_ts14(trade: Dict[str, Any]) -> str:
|
||||
for k in ("buy_time", "entry_time", "buy_date"):
|
||||
t = _ts14(trade.get(k))
|
||||
if t:
|
||||
return t
|
||||
return ""
|
||||
|
||||
|
||||
def _exit_ts14(trade: Dict[str, Any]) -> str:
|
||||
for k in ("sell_time", "exit_time", "sell_date"):
|
||||
t = _ts14(trade.get(k))
|
||||
if t:
|
||||
return t
|
||||
return ""
|
||||
|
||||
|
||||
def _spread_pct(best_bid: float, best_ask: float) -> Optional[float]:
|
||||
if best_bid <= 0 or best_ask <= 0:
|
||||
return None
|
||||
mid = (best_bid + best_ask) / 2.0
|
||||
if mid <= 0:
|
||||
return None
|
||||
return (best_ask - best_bid) / mid * 100.0
|
||||
|
||||
|
||||
def _or_ratio(total_bid: float, total_ask: float) -> Optional[float]:
|
||||
try:
|
||||
ask = float(total_ask or 0)
|
||||
bid = float(total_bid or 0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if ask <= 0:
|
||||
return None
|
||||
return bid / ask
|
||||
|
||||
|
||||
def _row_to_ob(
|
||||
row: Dict[str, Any],
|
||||
*,
|
||||
delta_sec: int,
|
||||
lim_spread_pct: float,
|
||||
lim_ratio: float,
|
||||
lim_or_ratio: float,
|
||||
side: str,
|
||||
) -> Dict[str, Any]:
|
||||
bid = float(row.get("best_bid") or 0)
|
||||
ask = float(row.get("best_ask") or 0)
|
||||
bid_l3 = int(row.get("bid_qty_l3") or 0)
|
||||
ask_l3 = int(row.get("ask_qty_l3") or 0)
|
||||
ratio = (bid_l3 / ask_l3) if ask_l3 > 0 else None
|
||||
tot_bid = int(row.get("total_bid_qty") or 0)
|
||||
tot_ask = int(row.get("total_ask_qty") or 0)
|
||||
or_r = _or_ratio(tot_bid, tot_ask)
|
||||
rej = (row.get("reject_code") or "").strip() or None
|
||||
msg = (row.get("reject_msg") or "").strip() or None
|
||||
src = (row.get("source") or "").strip()
|
||||
if side == "entry" and src == "filter_eval":
|
||||
verdict = rej or "PASS"
|
||||
else:
|
||||
verdict = "BODY" # 주기 스냅 — 판정 메타 없음 (매도도 본체 위주)
|
||||
near_only = int(delta_sec) > _STRICT_DELTA_SEC
|
||||
return {
|
||||
"side": side,
|
||||
"snap_time": str(row.get("snap_time") or "")[:14],
|
||||
"source": src,
|
||||
"strategy": row.get("strategy"),
|
||||
"best_bid": int(bid) if bid else 0,
|
||||
"best_ask": int(ask) if ask else 0,
|
||||
"spread_pct": round(_spread_pct(bid, ask) or 0.0, 3),
|
||||
"bid_qty_l3": bid_l3,
|
||||
"ask_qty_l3": ask_l3,
|
||||
"ratio": round(ratio, 3) if ratio is not None else None,
|
||||
"total_bid_qty": tot_bid,
|
||||
"total_ask_qty": tot_ask,
|
||||
"or_ratio": round(or_r, 3) if or_r is not None else None,
|
||||
"reject_code": rej,
|
||||
"reject_msg": msg,
|
||||
"verdict": verdict,
|
||||
"delta_sec": int(delta_sec),
|
||||
"matched": True,
|
||||
"near_only": bool(near_only),
|
||||
"lim_spread_pct": float(lim_spread_pct),
|
||||
"lim_ratio": float(lim_ratio),
|
||||
"lim_or_ratio": float(lim_or_ratio),
|
||||
}
|
||||
|
||||
|
||||
def _pick_best(
|
||||
candidates: Sequence[Tuple[int, Dict[str, Any]]],
|
||||
*,
|
||||
prefer_strategy: str,
|
||||
prefer_body: bool = False,
|
||||
) -> Optional[Tuple[int, Dict[str, Any]]]:
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
def _fe(c: Tuple[int, Dict[str, Any]]) -> bool:
|
||||
return (c[1].get("source") or "") == "filter_eval"
|
||||
|
||||
def _is_body(c: Tuple[int, Dict[str, Any]]) -> bool:
|
||||
# filter_eval = 진입 TRIGGER 판정 스냅 · 그 외(kiwoom_0d·ls_*)는 본체
|
||||
return (c[1].get("source") or "") != "filter_eval"
|
||||
|
||||
# 매도: filter_eval(진입판정)보다 본체 시계열 우선
|
||||
if prefer_body:
|
||||
body_pref = [c for c in candidates if _is_body(c) and c[0] <= _BODY_PREF_DELTA_SEC]
|
||||
if body_pref:
|
||||
return min(body_pref, key=lambda x: x[0])
|
||||
body_soft = [c for c in candidates if _is_body(c) and c[0] <= _SOFT_DELTA_SEC]
|
||||
if body_soft:
|
||||
return min(body_soft, key=lambda x: x[0])
|
||||
fe_any = [c for c in candidates if _fe(c) and c[0] <= _SOFT_DELTA_SEC]
|
||||
if fe_any:
|
||||
return min(fe_any, key=lambda x: x[0])
|
||||
return None
|
||||
|
||||
# 매수: 기존 우선순위 (filter_eval → 본체)
|
||||
fe_match = [
|
||||
c for c in candidates
|
||||
if _fe(c)
|
||||
and c[0] <= _STRICT_DELTA_SEC
|
||||
and _strategy_canon(str(c[1].get("strategy") or "")) == prefer_strategy
|
||||
]
|
||||
if fe_match:
|
||||
return min(fe_match, key=lambda x: x[0])
|
||||
fe_any = [c for c in candidates if _fe(c) and c[0] <= _STRICT_DELTA_SEC]
|
||||
if fe_any:
|
||||
return min(fe_any, key=lambda x: x[0])
|
||||
body_pref = [c for c in candidates if _is_body(c) and c[0] <= _BODY_PREF_DELTA_SEC]
|
||||
if body_pref:
|
||||
return min(body_pref, key=lambda x: x[0])
|
||||
fe_soft = [c for c in candidates if _fe(c) and c[0] <= _SOFT_DELTA_SEC]
|
||||
if fe_soft:
|
||||
return min(fe_soft, key=lambda x: x[0])
|
||||
body_soft = [c for c in candidates if _is_body(c) and c[0] <= _SOFT_DELTA_SEC]
|
||||
if body_soft:
|
||||
return min(body_soft, key=lambda x: x[0])
|
||||
return None
|
||||
|
||||
|
||||
def _load_lims(prefer: str) -> Tuple[float, float, float]:
|
||||
lim_spread = 0.45
|
||||
lim_ratio = 0.85
|
||||
lim_or = float(_EXIT_OB_OR_DEFAULT)
|
||||
try:
|
||||
from kis_trader.engine.orderbook_env import (
|
||||
OB_DEFAULT_MAX_SPREAD_PCT,
|
||||
OB_DEFAULT_MIN_BID_ASK_RATIO,
|
||||
load_orderbook_threshold_cfg as _load_ob_cfg,
|
||||
)
|
||||
lim_spread = float(OB_DEFAULT_MAX_SPREAD_PCT)
|
||||
lim_ratio = float(OB_DEFAULT_MIN_BID_ASK_RATIO)
|
||||
if prefer and prefer != "US":
|
||||
cfg = _load_ob_cfg(prefer)
|
||||
lim_spread = float(cfg.get("max_spread_pct") or lim_spread)
|
||||
lim_ratio = float(cfg.get("min_bid_ask_ratio") or lim_ratio)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from kis_trader.utils.env import get_env_float
|
||||
lim_or = float(get_env_float("MOMENTUM_EXIT_OB_RATIO_MIN", _EXIT_OB_OR_DEFAULT))
|
||||
except Exception:
|
||||
pass
|
||||
return lim_spread, lim_ratio, lim_or
|
||||
|
||||
|
||||
def _fetch_ob_rows(
|
||||
db: Any,
|
||||
code_list: List[str],
|
||||
lo: str,
|
||||
hi: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
conn = getattr(db, "conn", None) or db
|
||||
placeholders = ",".join(["%s"] * len(code_list))
|
||||
params: List[Any] = ["KR", *code_list, lo, hi]
|
||||
sql_kw = (
|
||||
f"SELECT code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty, "
|
||||
f"bid_qty_l3, ask_qty_l3, source, strategy, reject_code, reject_msg "
|
||||
f"FROM ws_orderbook WHERE market = %s AND code IN ({placeholders}) "
|
||||
f"AND snap_time >= %s AND snap_time <= %s "
|
||||
f"AND source IN ('filter_eval', 'kiwoom_0d') "
|
||||
f"ORDER BY code, snap_time"
|
||||
)
|
||||
rows = [dict(r) for r in conn.execute(sql_kw, params).fetchall()]
|
||||
sql_ls = (
|
||||
f"SELECT code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty, "
|
||||
f"bid_qty_l3, ask_qty_l3, source, "
|
||||
f"NULL AS strategy, NULL AS reject_code, NULL AS reject_msg "
|
||||
f"FROM ls_ws_orderbook WHERE market = %s AND code IN ({placeholders}) "
|
||||
f"AND snap_time >= %s AND snap_time <= %s "
|
||||
f"ORDER BY code, snap_time"
|
||||
)
|
||||
try:
|
||||
ls_rows = [dict(r) for r in conn.execute(sql_ls, params).fetchall()]
|
||||
if ls_rows:
|
||||
rows.extend(ls_rows)
|
||||
rows.sort(
|
||||
key=lambda r: (
|
||||
str(r.get("code") or ""),
|
||||
str(r.get("snap_time") or ""),
|
||||
)
|
||||
)
|
||||
except Exception as e_ls:
|
||||
logger.debug("ls_ws_orderbook 조회 스킵: %s", e_ls)
|
||||
return rows
|
||||
|
||||
|
||||
def enrich_trades_with_entry_orderbook(
|
||||
db: Any,
|
||||
trades: List[Dict[str, Any]],
|
||||
*,
|
||||
strategy_hint: str = "",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""하위호환 — 매수·매도 호가 모두 부착."""
|
||||
return enrich_trades_with_orderbook(db, trades, strategy_hint=strategy_hint)
|
||||
|
||||
|
||||
def enrich_trades_with_orderbook(
|
||||
db: Any,
|
||||
trades: List[Dict[str, Any]],
|
||||
*,
|
||||
strategy_hint: str = "",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""거래 dict 에 ``entry_ob`` · ``exit_ob`` 를 in-place 부착."""
|
||||
if not trades:
|
||||
return trades
|
||||
|
||||
hint = _strategy_canon(strategy_hint)
|
||||
if hint == "US":
|
||||
for t in trades:
|
||||
t["entry_ob"] = None
|
||||
t["exit_ob"] = None
|
||||
return trades
|
||||
|
||||
# (trade_idx, trade, code, entry_dt|None, exit_dt|None)
|
||||
keyed: List[Tuple[int, Dict[str, Any], str, Optional[datetime], Optional[datetime]]] = []
|
||||
codes = set()
|
||||
t_min: Optional[datetime] = None
|
||||
t_max: Optional[datetime] = None
|
||||
|
||||
def _bump(dt: Optional[datetime]) -> None:
|
||||
nonlocal t_min, t_max
|
||||
if dt is None:
|
||||
return
|
||||
if t_min is None or dt < t_min:
|
||||
t_min = dt
|
||||
if t_max is None or dt > t_max:
|
||||
t_max = dt
|
||||
|
||||
for i, t in enumerate(trades):
|
||||
if _strategy_canon(str(t.get("strategy") or strategy_hint)) == "US":
|
||||
t["entry_ob"] = None
|
||||
t["exit_ob"] = None
|
||||
continue
|
||||
code = str(t.get("code") or "").strip()
|
||||
edt = _parse14(_entry_ts14(t))
|
||||
xdt = _parse14(_exit_ts14(t))
|
||||
if not code or (edt is None and xdt is None):
|
||||
t["entry_ob"] = None
|
||||
t["exit_ob"] = None
|
||||
continue
|
||||
keyed.append((i, t, code, edt, xdt))
|
||||
codes.add(code)
|
||||
_bump(edt)
|
||||
_bump(xdt)
|
||||
|
||||
if not keyed or t_min is None or t_max is None:
|
||||
for t in trades:
|
||||
t.setdefault("entry_ob", None)
|
||||
t.setdefault("exit_ob", None)
|
||||
return trades
|
||||
|
||||
lo = (t_min - timedelta(seconds=_SOFT_DELTA_SEC)).strftime("%Y%m%d%H%M%S")
|
||||
hi = (t_max + timedelta(seconds=_SOFT_DELTA_SEC)).strftime("%Y%m%d%H%M%S")
|
||||
code_list = sorted(codes)
|
||||
|
||||
try:
|
||||
rows = _fetch_ob_rows(db, code_list, lo, hi)
|
||||
except Exception as e:
|
||||
logger.warning("orderbook enrich 조회 실패: %s", e)
|
||||
for t in trades:
|
||||
t.setdefault("entry_ob", None)
|
||||
t.setdefault("exit_ob", None)
|
||||
return trades
|
||||
|
||||
by_code: Dict[str, List[Dict[str, Any]]] = {}
|
||||
for r in rows:
|
||||
c = str(r.get("code") or "").strip()
|
||||
if not c:
|
||||
continue
|
||||
by_code.setdefault(c, []).append(r)
|
||||
|
||||
for _i, t, code, edt, xdt in keyed:
|
||||
prefer = _strategy_canon(str(t.get("strategy") or strategy_hint) or hint)
|
||||
lim_spread, lim_ratio, lim_or = _load_lims(prefer)
|
||||
|
||||
def _cands_for(dt: Optional[datetime]) -> List[Tuple[int, Dict[str, Any]]]:
|
||||
if dt is None:
|
||||
return []
|
||||
out: List[Tuple[int, Dict[str, Any]]] = []
|
||||
for r in by_code.get(code, []):
|
||||
rdt = _parse14(str(r.get("snap_time") or ""))
|
||||
if rdt is None:
|
||||
continue
|
||||
delta = abs(int((rdt - dt).total_seconds()))
|
||||
if delta > _SOFT_DELTA_SEC:
|
||||
continue
|
||||
out.append((delta, r))
|
||||
return out
|
||||
|
||||
ep = _pick_best(_cands_for(edt), prefer_strategy=prefer, prefer_body=False)
|
||||
if ep:
|
||||
delta, row = ep
|
||||
t["entry_ob"] = _row_to_ob(
|
||||
row,
|
||||
delta_sec=delta,
|
||||
lim_spread_pct=lim_spread,
|
||||
lim_ratio=lim_ratio,
|
||||
lim_or_ratio=lim_or,
|
||||
side="entry",
|
||||
)
|
||||
else:
|
||||
t["entry_ob"] = None
|
||||
|
||||
# 미청산(보유중) — 매도호가 없음
|
||||
if xdt is None:
|
||||
t["exit_ob"] = None
|
||||
else:
|
||||
xp = _pick_best(_cands_for(xdt), prefer_strategy=prefer, prefer_body=True)
|
||||
if xp:
|
||||
delta, row = xp
|
||||
t["exit_ob"] = _row_to_ob(
|
||||
row,
|
||||
delta_sec=delta,
|
||||
lim_spread_pct=lim_spread,
|
||||
lim_ratio=lim_ratio,
|
||||
lim_or_ratio=lim_or,
|
||||
side="exit",
|
||||
)
|
||||
else:
|
||||
t["exit_ob"] = None
|
||||
|
||||
for t in trades:
|
||||
t.setdefault("entry_ob", None)
|
||||
t.setdefault("exit_ob", None)
|
||||
return trades
|
||||
Reference in New Issue
Block a user