2c7ad867f4
feat: Implement backtest source management and enhance candle data handling Changes: - Introduced a new function `_apply_backtest_source_env_from_request` to manage the environment variables for candle, tick, and order book sources based on incoming requests. - Added a teardown function `_teardown_backtest_source_env` to ensure that environment variables do not persist between requests, enhancing the stability of the backtesting environment. - Refactored existing code to utilize the new source management functions, improving code readability and maintainability. - Added new utility functions in `bt_candle_source.py` for fetching and managing candle data, ensuring consistency with live trading data sources. Impact: - These changes improve the flexibility and reliability of the backtesting framework, allowing for better management of data sources and reducing the risk of cross-request contamination.
420 lines
14 KiB
Python
420 lines
14 KiB
Python
"""
|
|
거래내역 UI용 — 매수/매도 시각 근처 호가 스냅 부착.
|
|
|
|
필터 ON/OFF 와 무관: 수집된 ws_orderbook / ls_ws_orderbook 으로
|
|
진입(매수) · 청산(매도) 시점 유동성을 각각 보여 준다.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
|
|
logger = logging.getLogger("trade_orderbook_enrich")
|
|
|
|
# 표시용 매칭 창 — 체결≠호가저장 이므로 실매 UI는 ±15분까지 근처 기록 허용
|
|
_STRICT_DELTA_SEC = 180 # 판정(filter_eval) 정밀 창
|
|
_SOFT_DELTA_SEC = 900 # 본체·표시용 완화 창 (±15분)
|
|
_BODY_PREF_DELTA_SEC = 120 # 본체 우선 창
|
|
_EXIT_OB_OR_DEFAULT = 0.4 # L3 OR 임계 표시용 (MOMENTUM_EXIT_OB_RATIO_MIN 기본)
|
|
|
|
|
|
def _strategy_canon(strategy: str) -> str:
|
|
s = (strategy or "").strip().upper()
|
|
if s in ("SHORT", "TAIL_CATCH", "TAIL"):
|
|
return "TAIL"
|
|
if s in ("BO", "BREAKOUT"):
|
|
return "BREAKOUT"
|
|
if s in ("MOM", "MOMENTUM"):
|
|
return "MOMENTUM"
|
|
if s in ("SCALP", "SCALPING", "REVERSAL"):
|
|
return "SCALP"
|
|
if s.startswith("US_"):
|
|
return "US"
|
|
return s
|
|
|
|
|
|
def _ts14(raw: Any) -> str:
|
|
if raw is None:
|
|
return ""
|
|
s = str(raw).strip()
|
|
if not s:
|
|
return ""
|
|
s = (
|
|
s.replace("-", "")
|
|
.replace(":", "")
|
|
.replace(" ", "")
|
|
.replace("T", "")
|
|
.replace(".", "")
|
|
)
|
|
if len(s) < 8:
|
|
return ""
|
|
return (s + "000000")[:14]
|
|
|
|
|
|
def _parse14(ts14: str) -> Optional[datetime]:
|
|
st = (ts14 or "").strip()
|
|
if len(st) < 12:
|
|
return None
|
|
try:
|
|
return datetime.strptime(st[:14], "%Y%m%d%H%M%S")
|
|
except ValueError:
|
|
try:
|
|
return datetime.strptime(st[:12], "%Y%m%d%H%M")
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _entry_ts14(trade: Dict[str, Any]) -> str:
|
|
for k in ("buy_time", "entry_time", "buy_date"):
|
|
t = _ts14(trade.get(k))
|
|
if t:
|
|
return t
|
|
return ""
|
|
|
|
|
|
def _exit_ts14(trade: Dict[str, Any]) -> str:
|
|
for k in ("sell_time", "exit_time", "sell_date"):
|
|
t = _ts14(trade.get(k))
|
|
if t:
|
|
return t
|
|
return ""
|
|
|
|
|
|
def _spread_pct(best_bid: float, best_ask: float) -> Optional[float]:
|
|
if best_bid <= 0 or best_ask <= 0:
|
|
return None
|
|
mid = (best_bid + best_ask) / 2.0
|
|
if mid <= 0:
|
|
return None
|
|
return (best_ask - best_bid) / mid * 100.0
|
|
|
|
|
|
def _or_ratio(total_bid: float, total_ask: float) -> Optional[float]:
|
|
try:
|
|
ask = float(total_ask or 0)
|
|
bid = float(total_bid or 0)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if ask <= 0:
|
|
return None
|
|
return bid / ask
|
|
|
|
|
|
def _row_to_ob(
|
|
row: Dict[str, Any],
|
|
*,
|
|
delta_sec: int,
|
|
lim_spread_pct: float,
|
|
lim_ratio: float,
|
|
lim_or_ratio: float,
|
|
side: str,
|
|
) -> Dict[str, Any]:
|
|
bid = float(row.get("best_bid") or 0)
|
|
ask = float(row.get("best_ask") or 0)
|
|
bid_l3 = int(row.get("bid_qty_l3") or 0)
|
|
ask_l3 = int(row.get("ask_qty_l3") or 0)
|
|
ratio = (bid_l3 / ask_l3) if ask_l3 > 0 else None
|
|
tot_bid = int(row.get("total_bid_qty") or 0)
|
|
tot_ask = int(row.get("total_ask_qty") or 0)
|
|
or_r = _or_ratio(tot_bid, tot_ask)
|
|
rej = (row.get("reject_code") or "").strip() or None
|
|
msg = (row.get("reject_msg") or "").strip() or None
|
|
src = (row.get("source") or "").strip()
|
|
if side == "entry" and src == "filter_eval":
|
|
verdict = rej or "PASS"
|
|
else:
|
|
verdict = "BODY" # 주기 스냅 — 판정 메타 없음 (매도도 본체 위주)
|
|
near_only = int(delta_sec) > _STRICT_DELTA_SEC
|
|
# ratio = L3 매수/매도 잔량비, or_ratio = 전체 잔량비
|
|
# mid_ratio/whale_ratio = 웹 표(M/W) 호환 별칭 (동일 값)
|
|
ratio_r = round(ratio, 3) if ratio is not None else None
|
|
or_r_r = round(or_r, 3) if or_r is not None else None
|
|
return {
|
|
"side": side,
|
|
"snap_time": str(row.get("snap_time") or "")[:14],
|
|
"source": src,
|
|
"strategy": row.get("strategy"),
|
|
"best_bid": int(bid) if bid else 0,
|
|
"best_ask": int(ask) if ask else 0,
|
|
"spread_pct": round(_spread_pct(bid, ask) or 0.0, 3),
|
|
"bid_qty_l3": bid_l3,
|
|
"ask_qty_l3": ask_l3,
|
|
"ratio": ratio_r,
|
|
"mid_ratio": ratio_r,
|
|
"total_bid_qty": tot_bid,
|
|
"total_ask_qty": tot_ask,
|
|
"or_ratio": or_r_r,
|
|
"whale_ratio": or_r_r,
|
|
"reject_code": rej,
|
|
"reject_msg": msg,
|
|
"verdict": verdict,
|
|
"delta_sec": int(delta_sec),
|
|
"matched": True,
|
|
"near_only": bool(near_only),
|
|
"lim_spread_pct": float(lim_spread_pct),
|
|
"lim_ratio": float(lim_ratio),
|
|
"lim_or_ratio": float(lim_or_ratio),
|
|
}
|
|
|
|
|
|
def _pick_best(
|
|
candidates: Sequence[Tuple[int, Dict[str, Any]]],
|
|
*,
|
|
prefer_strategy: str,
|
|
prefer_body: bool = False,
|
|
) -> Optional[Tuple[int, Dict[str, Any]]]:
|
|
if not candidates:
|
|
return None
|
|
|
|
def _fe(c: Tuple[int, Dict[str, Any]]) -> bool:
|
|
return (c[1].get("source") or "") == "filter_eval"
|
|
|
|
def _is_body(c: Tuple[int, Dict[str, Any]]) -> bool:
|
|
# filter_eval = 진입 TRIGGER 판정 스냅 · 그 외(kiwoom_0d·ls_*)는 본체
|
|
return (c[1].get("source") or "") != "filter_eval"
|
|
|
|
# 매도: filter_eval(진입판정)보다 본체 시계열 우선
|
|
if prefer_body:
|
|
body_pref = [c for c in candidates if _is_body(c) and c[0] <= _BODY_PREF_DELTA_SEC]
|
|
if body_pref:
|
|
return min(body_pref, key=lambda x: x[0])
|
|
body_soft = [c for c in candidates if _is_body(c) and c[0] <= _SOFT_DELTA_SEC]
|
|
if body_soft:
|
|
return min(body_soft, key=lambda x: x[0])
|
|
fe_any = [c for c in candidates if _fe(c) and c[0] <= _SOFT_DELTA_SEC]
|
|
if fe_any:
|
|
return min(fe_any, key=lambda x: x[0])
|
|
return None
|
|
|
|
# 매수: 기존 우선순위 (filter_eval → 본체)
|
|
fe_match = [
|
|
c for c in candidates
|
|
if _fe(c)
|
|
and c[0] <= _STRICT_DELTA_SEC
|
|
and _strategy_canon(str(c[1].get("strategy") or "")) == prefer_strategy
|
|
]
|
|
if fe_match:
|
|
return min(fe_match, key=lambda x: x[0])
|
|
fe_any = [c for c in candidates if _fe(c) and c[0] <= _STRICT_DELTA_SEC]
|
|
if fe_any:
|
|
return min(fe_any, key=lambda x: x[0])
|
|
body_pref = [c for c in candidates if _is_body(c) and c[0] <= _BODY_PREF_DELTA_SEC]
|
|
if body_pref:
|
|
return min(body_pref, key=lambda x: x[0])
|
|
fe_soft = [c for c in candidates if _fe(c) and c[0] <= _SOFT_DELTA_SEC]
|
|
if fe_soft:
|
|
return min(fe_soft, key=lambda x: x[0])
|
|
body_soft = [c for c in candidates if _is_body(c) and c[0] <= _SOFT_DELTA_SEC]
|
|
if body_soft:
|
|
return min(body_soft, key=lambda x: x[0])
|
|
return None
|
|
|
|
|
|
def _load_lims(prefer: str) -> Tuple[float, float, float]:
|
|
lim_spread = 0.45
|
|
lim_ratio = 0.85
|
|
lim_or = float(_EXIT_OB_OR_DEFAULT)
|
|
try:
|
|
from kis_trader.engine.orderbook_env import (
|
|
OB_DEFAULT_MAX_SPREAD_PCT,
|
|
OB_DEFAULT_MIN_BID_ASK_RATIO,
|
|
load_orderbook_threshold_cfg as _load_ob_cfg,
|
|
)
|
|
lim_spread = float(OB_DEFAULT_MAX_SPREAD_PCT)
|
|
lim_ratio = float(OB_DEFAULT_MIN_BID_ASK_RATIO)
|
|
if prefer and prefer != "US":
|
|
cfg = _load_ob_cfg(prefer)
|
|
lim_spread = float(cfg.get("max_spread_pct") or lim_spread)
|
|
lim_ratio = float(cfg.get("min_bid_ask_ratio") or lim_ratio)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
from kis_trader.utils.env import get_env_float
|
|
lim_or = float(get_env_float("MOMENTUM_EXIT_OB_RATIO_MIN", _EXIT_OB_OR_DEFAULT))
|
|
except Exception:
|
|
pass
|
|
return lim_spread, lim_ratio, lim_or
|
|
|
|
|
|
def _fetch_ob_rows(
|
|
db: Any,
|
|
code_list: List[str],
|
|
lo: str,
|
|
hi: str,
|
|
) -> List[Dict[str, Any]]:
|
|
conn = getattr(db, "conn", None) or db
|
|
placeholders = ",".join(["%s"] * len(code_list))
|
|
params: List[Any] = ["KR", *code_list, lo, hi]
|
|
sql_kw = (
|
|
f"SELECT code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty, "
|
|
f"bid_qty_l3, ask_qty_l3, source, strategy, reject_code, reject_msg "
|
|
f"FROM ws_orderbook WHERE market = %s AND code IN ({placeholders}) "
|
|
f"AND snap_time >= %s AND snap_time <= %s "
|
|
f"AND source IN ('filter_eval', 'kiwoom_0d') "
|
|
f"ORDER BY code, snap_time"
|
|
)
|
|
rows = [dict(r) for r in conn.execute(sql_kw, params).fetchall()]
|
|
sql_ls = (
|
|
f"SELECT code, snap_time, best_bid, best_ask, total_bid_qty, total_ask_qty, "
|
|
f"bid_qty_l3, ask_qty_l3, source, "
|
|
f"NULL AS strategy, NULL AS reject_code, NULL AS reject_msg "
|
|
f"FROM ls_ws_orderbook WHERE market = %s AND code IN ({placeholders}) "
|
|
f"AND snap_time >= %s AND snap_time <= %s "
|
|
f"ORDER BY code, snap_time"
|
|
)
|
|
try:
|
|
ls_rows = [dict(r) for r in conn.execute(sql_ls, params).fetchall()]
|
|
if ls_rows:
|
|
rows.extend(ls_rows)
|
|
rows.sort(
|
|
key=lambda r: (
|
|
str(r.get("code") or ""),
|
|
str(r.get("snap_time") or ""),
|
|
)
|
|
)
|
|
except Exception as e_ls:
|
|
logger.debug("ls_ws_orderbook 조회 스킵: %s", e_ls)
|
|
return rows
|
|
|
|
|
|
def enrich_trades_with_entry_orderbook(
|
|
db: Any,
|
|
trades: List[Dict[str, Any]],
|
|
*,
|
|
strategy_hint: str = "",
|
|
) -> List[Dict[str, Any]]:
|
|
"""하위호환 — 매수·매도 호가 모두 부착."""
|
|
return enrich_trades_with_orderbook(db, trades, strategy_hint=strategy_hint)
|
|
|
|
|
|
def enrich_trades_with_orderbook(
|
|
db: Any,
|
|
trades: List[Dict[str, Any]],
|
|
*,
|
|
strategy_hint: str = "",
|
|
) -> List[Dict[str, Any]]:
|
|
"""거래 dict 에 ``entry_ob`` · ``exit_ob`` 를 in-place 부착."""
|
|
if not trades:
|
|
return trades
|
|
|
|
hint = _strategy_canon(strategy_hint)
|
|
if hint == "US":
|
|
for t in trades:
|
|
t["entry_ob"] = None
|
|
t["exit_ob"] = None
|
|
return trades
|
|
|
|
# (trade_idx, trade, code, entry_dt|None, exit_dt|None)
|
|
keyed: List[Tuple[int, Dict[str, Any], str, Optional[datetime], Optional[datetime]]] = []
|
|
codes = set()
|
|
t_min: Optional[datetime] = None
|
|
t_max: Optional[datetime] = None
|
|
|
|
def _bump(dt: Optional[datetime]) -> None:
|
|
nonlocal t_min, t_max
|
|
if dt is None:
|
|
return
|
|
if t_min is None or dt < t_min:
|
|
t_min = dt
|
|
if t_max is None or dt > t_max:
|
|
t_max = dt
|
|
|
|
for i, t in enumerate(trades):
|
|
if _strategy_canon(str(t.get("strategy") or strategy_hint)) == "US":
|
|
t["entry_ob"] = None
|
|
t["exit_ob"] = None
|
|
continue
|
|
code = str(t.get("code") or "").strip()
|
|
edt = _parse14(_entry_ts14(t))
|
|
xdt = _parse14(_exit_ts14(t))
|
|
if not code or (edt is None and xdt is None):
|
|
t["entry_ob"] = None
|
|
t["exit_ob"] = None
|
|
continue
|
|
keyed.append((i, t, code, edt, xdt))
|
|
codes.add(code)
|
|
_bump(edt)
|
|
_bump(xdt)
|
|
|
|
if not keyed or t_min is None or t_max is None:
|
|
for t in trades:
|
|
t.setdefault("entry_ob", None)
|
|
t.setdefault("exit_ob", None)
|
|
return trades
|
|
|
|
lo = (t_min - timedelta(seconds=_SOFT_DELTA_SEC)).strftime("%Y%m%d%H%M%S")
|
|
hi = (t_max + timedelta(seconds=_SOFT_DELTA_SEC)).strftime("%Y%m%d%H%M%S")
|
|
code_list = sorted(codes)
|
|
|
|
try:
|
|
rows = _fetch_ob_rows(db, code_list, lo, hi)
|
|
except Exception as e:
|
|
logger.warning("orderbook enrich 조회 실패: %s", e)
|
|
for t in trades:
|
|
t.setdefault("entry_ob", None)
|
|
t.setdefault("exit_ob", None)
|
|
return trades
|
|
|
|
by_code: Dict[str, List[Dict[str, Any]]] = {}
|
|
for r in rows:
|
|
c = str(r.get("code") or "").strip()
|
|
if not c:
|
|
continue
|
|
by_code.setdefault(c, []).append(r)
|
|
|
|
for _i, t, code, edt, xdt in keyed:
|
|
prefer = _strategy_canon(str(t.get("strategy") or strategy_hint) or hint)
|
|
lim_spread, lim_ratio, lim_or = _load_lims(prefer)
|
|
|
|
def _cands_for(dt: Optional[datetime]) -> List[Tuple[int, Dict[str, Any]]]:
|
|
if dt is None:
|
|
return []
|
|
out: List[Tuple[int, Dict[str, Any]]] = []
|
|
for r in by_code.get(code, []):
|
|
rdt = _parse14(str(r.get("snap_time") or ""))
|
|
if rdt is None:
|
|
continue
|
|
delta = abs(int((rdt - dt).total_seconds()))
|
|
if delta > _SOFT_DELTA_SEC:
|
|
continue
|
|
out.append((delta, r))
|
|
return out
|
|
|
|
ep = _pick_best(_cands_for(edt), prefer_strategy=prefer, prefer_body=False)
|
|
if ep:
|
|
delta, row = ep
|
|
t["entry_ob"] = _row_to_ob(
|
|
row,
|
|
delta_sec=delta,
|
|
lim_spread_pct=lim_spread,
|
|
lim_ratio=lim_ratio,
|
|
lim_or_ratio=lim_or,
|
|
side="entry",
|
|
)
|
|
else:
|
|
t["entry_ob"] = None
|
|
|
|
# 미청산(보유중) — 매도호가 없음
|
|
if xdt is None:
|
|
t["exit_ob"] = None
|
|
else:
|
|
xp = _pick_best(_cands_for(xdt), prefer_strategy=prefer, prefer_body=True)
|
|
if xp:
|
|
delta, row = xp
|
|
t["exit_ob"] = _row_to_ob(
|
|
row,
|
|
delta_sec=delta,
|
|
lim_spread_pct=lim_spread,
|
|
lim_ratio=lim_ratio,
|
|
lim_or_ratio=lim_or,
|
|
side="exit",
|
|
)
|
|
else:
|
|
t["exit_ob"] = None
|
|
|
|
for t in trades:
|
|
t.setdefault("entry_ob", None)
|
|
t.setdefault("exit_ob", None)
|
|
return trades
|