Files
kis_bot/kis_trader/utils/live_portfolio_common.py
Your Name 2c7ad867f4 거래 빠르게 안티에서 병신만든거 커서로
feat: Implement backtest source management and enhance candle data handling

Changes:
- Introduced a new function `_apply_backtest_source_env_from_request` to manage the environment variables for candle, tick, and order book sources based on incoming requests.
- Added a teardown function `_teardown_backtest_source_env` to ensure that environment variables do not persist between requests, enhancing the stability of the backtesting environment.
- Refactored existing code to utilize the new source management functions, improving code readability and maintainability.
- Added new utility functions in `bt_candle_source.py` for fetching and managing candle data, ensuring consistency with live trading data sources.

Impact:
- These changes improve the flexibility and reliability of the backtesting framework, allowing for better management of data sources and reducing the risk of cross-request contamination.
2026-08-13 16:03:40 +09:00

615 lines
21 KiB
Python

"""
실매 ↔ 포트폴리오 백테 정합 — 유니버스 슬롯·총한도·매수금액.
백테 ``breakout_portfolio_backtest`` / ``backtest_portfolio_common`` 과 동일 분모.
"""
from __future__ import annotations
import threading
import time
from datetime import datetime as dt
from typing import Any, Dict, List, Optional, Set, Tuple
from ..backtest.backtest_portfolio_common import (
STRATEGY_PORTFOLIO_KEYS,
min_invest_ratio_of_slot,
portfolio_exposure_krw,
resolve_portfolio_params,
target_qty_and_cost,
)
from ..utils.env import get_env_bool, get_env_float, get_env_from_db, get_env_int
from ..utils.position_sizing import invest_qty_for_price
# 자체 유니버스 — 기본 슬롯 필터 OFF (env 로 켤 수 있음)
_UNIVERSE_SLOT_SKIP_DEFAULT = frozenset({"UPDOW", "DBBAND"})
# 총한도·일1회 base 가드 스킵 (전략 자체 구현)
_PORTFOLIO_BUDGET_SKIP_DEFAULT = frozenset({"UPDOW"})
# 유니버스 슬롯 history — RAM TTL (정합 ON 유지 · 루프마다 SELECT 금지)
_SLOT_UNIV_LOCK = threading.Lock()
# key=(sid, src, slot_key) → (codes|None, ts). codes=None → 스냅샷 없음(필터 생략)
_SLOT_UNIV_CACHE: Dict[Tuple[str, str, str], Tuple[Optional[Set[str]], float]] = {}
# 같은 키 TTL 만료 시 DB SELECT 1회만 (대기자는 Event 공유) — 분슬롯 freeze 아님
_SLOT_UNIV_FLIGHT: Dict[Tuple[str, str, str], threading.Event] = {}
def _slot_univ_ttl_sec() -> float:
try:
return float(get_env_float("UNIVERSE_SLOT_CACHE_TTL_SEC", 1.0) or 1.0)
except Exception:
return 1.0
def _slot_univ_flight_wait_sec() -> float:
"""single-flight 대기 상한(초). 리더 실패/타임아웃 후 재시도용."""
try:
return float(get_env_float("UNIVERSE_SLOT_FLIGHT_WAIT_SEC", 5.0) or 5.0)
except Exception:
return 5.0
def invalidate_universe_slot_cache(strategy_id: Optional[str] = None) -> None:
"""슬롯 캐시 무효화. strategy_id 없으면 전체. in-flight 대기자도 깨움."""
sid = (strategy_id or "").upper()
wake: List[threading.Event] = []
with _SLOT_UNIV_LOCK:
if not sid:
_SLOT_UNIV_CACHE.clear()
wake = list(_SLOT_UNIV_FLIGHT.values())
_SLOT_UNIV_FLIGHT.clear()
else:
for k in list(_SLOT_UNIV_CACHE.keys()):
if k[0] == sid:
del _SLOT_UNIV_CACHE[k]
for k in list(_SLOT_UNIV_FLIGHT.keys()):
if k[0] == sid:
wake.append(_SLOT_UNIV_FLIGHT.pop(k))
for ev in wake:
try:
ev.set()
except Exception:
pass
def _prof_add(prof: Optional[Dict[str, float]], key: str, t0: float) -> None:
"""LOOP_PROF 세분용 — 매매 로직 불변, ms 누적만."""
if prof is None:
return
try:
prof[key] = float(prof.get(key, 0.0) or 0.0) + (time.perf_counter() - t0) * 1000.0
except Exception:
pass
def portfolio_strategy_key(strategy_id: str) -> str:
sid = (strategy_id or "").upper()
return "TAIL" if sid == "SHORT" else sid
def _env_flag(key: str, default: bool) -> bool:
raw = get_env_from_db(key, "")
if raw in (None, "", "None"):
return default
return str(raw).strip().lower() in ("1", "true", "t", "y", "yes", "on")
def live_universe_slot_align_enabled(strategy_id: str) -> bool:
"""실매 후보 ∩ history 슬롯 필터.
기본 **False** — history 지연/축소 시 전량탈락(universe_wipe) 방지.
백테 슬롯 정합이 필요하면 ``LIVE_UNIVERSE_SLOT_ALIGN=true`` 또는 전략별 키로 ON.
"""
sid = (strategy_id or "").upper()
per_key = f"{sid}_LIVE_UNIVERSE_SLOT_ALIGN"
if get_env_from_db(per_key, "") not in (None, "", "None"):
return _env_flag(per_key, False)
if sid in _UNIVERSE_SLOT_SKIP_DEFAULT:
return False
return _env_flag("LIVE_UNIVERSE_SLOT_ALIGN", False)
def resolve_live_universe_history_source(
strategy_id: str,
*,
universe_source: Optional[str] = None,
) -> str:
"""실매 슬롯 정합이 읽을 이력 테이블.
- ``ls_condition`` 유니버스 → ``ls_candidates_history``
- 그 외 → ``target_candidates_history`` (키움/KIS)
키움 이력과 LS 실후보를 교집합하면 전원 탈락(20→0) 한다.
"""
sid = (strategy_id or "").upper()
# 전략·전역 명시 오버라이드
for key in (
f"{sid}_LIVE_UNIVERSE_HISTORY_SOURCE",
"LIVE_UNIVERSE_HISTORY_SOURCE",
):
raw = get_env_from_db(key, "")
if raw not in (None, "", "None"):
s = str(raw).strip().lower()
if s in ("ls", "ls_condition", "ls_afr"):
return "ls"
if s in ("kiwoom", "target", "kis", "condition"):
return "kiwoom"
src = str(universe_source or "").strip().lower()
if not src:
src = str(
get_env_from_db(f"{sid}_UNIVERSE_SOURCE", "") or ""
).strip().lower()
if src in ("ls_condition", "ls", "ls_afr"):
return "ls"
return "kiwoom"
def live_portfolio_budget_align_enabled(strategy_id: str) -> bool:
sid = (strategy_id or "").upper()
per_key = f"{sid}_LIVE_PORTFOLIO_BUDGET_ALIGN"
if get_env_from_db(per_key, "") not in (None, "", "None"):
return _env_flag(per_key, True)
if sid in _PORTFOLIO_BUDGET_SKIP_DEFAULT:
return False
return _env_flag("LIVE_PORTFOLIO_BUDGET_ALIGN", True)
def resolve_live_scan_interval_min(strategy_id: str) -> int:
sid = (strategy_id or "").upper()
v = get_env_int(f"{sid}_SCAN_INTERVAL_MIN", 0)
if v > 0:
return max(1, int(v))
return max(1, get_env_int("SCAN_INTERVAL_MIN", 1))
def slot_key_from_dt(when: dt, scan_interval_min: int = 1) -> str:
"""백테 ``_bt_slot_key`` / ``_slot_key`` 와 동일."""
iv = max(1, int(scan_interval_min))
date = when.strftime("%Y%m%d")
hm = when.hour * 100 + when.minute
total_min = (hm // 100) * 60 + (hm % 100)
slot_min = (total_min // iv) * iv
slot_hm = (slot_min // 60) * 100 + (slot_min % 60)
return date + str(slot_hm).zfill(4)
def resolve_live_total_budget_krw(
strategy_id: str,
max_stocks: int,
slot_money: float,
) -> float:
pkey = portfolio_strategy_key(strategy_id)
keys = STRATEGY_PORTFOLIO_KEYS.get(pkey, STRATEGY_PORTFOLIO_KEYS["SCALP"])
env_row: Dict[str, Any] = {}
for k in keys.get("total_budget", ()):
v = get_env_int(k, 0)
if v > 0:
env_row[k] = v
break
for k in keys.get("slot", ()):
v = get_env_int(k, 0)
if v > 0:
env_row[k] = v
for k in keys.get("max_stocks", ()):
v = get_env_int(k, 0)
if v > 0:
env_row[k] = v
pf = resolve_portfolio_params(
env_row if env_row else None,
strategy=pkey,
slot_money=float(slot_money),
max_stocks=int(max_stocks),
)
return float(pf["total_budget_krw"])
def resolve_live_min_invest_ratio(strategy_id: str) -> float:
return min_invest_ratio_of_slot({}, strategy=portfolio_strategy_key(strategy_id))
def _history_snapshot_event_time(
db: Any,
strategy_id: str,
at_time: str,
history_source: str,
) -> Optional[Any]:
"""``at_time`` 이전 최신 ``event_time`` (없으면 None)."""
conn = getattr(db, "conn", None)
if conn is None:
return None
table_fn = getattr(db, "_universe_history_table", None)
if callable(table_fn):
try:
table = table_fn(history_source)
except Exception:
table = None
else:
table = None
if not table:
from kis_trader.backtest.universe_history_source import history_table_for_source
table = history_table_for_source(history_source)
try:
row = conn.execute(
f"""
SELECT MAX(event_time) AS et
FROM {table}
WHERE strategy_id=%s AND event_time <= %s
""",
(strategy_id, at_time),
).fetchone()
return (row or {}).get("et") if row else None
except Exception:
return None
def _event_time_ymd(et: Any) -> str:
if et is None:
return ""
if hasattr(et, "strftime"):
try:
return et.strftime("%Y-%m-%d")
except Exception:
pass
s = str(et).strip()
if len(s) >= 10 and s[4] == "-" and s[7] == "-":
return s[:10]
# YYYYMMDDHHMM / YYYYMMDD...
digits = "".join(ch for ch in s if ch.isdigit())
if len(digits) >= 8:
return f"{digits[:4]}-{digits[4:6]}-{digits[6:8]}"
return ""
def history_universe_codes_at(
db: Any,
strategy_id: str,
when: Optional[dt] = None,
*,
history_source: str = "kiwoom",
universe_source: Optional[str] = None,
prof: Optional[Dict[str, float]] = None,
) -> Optional[Set[str]]:
"""
이력 스냅샷 — ``at_time`` 이전 최신 event_time 종목 집합.
- kiwoom → ``target_candidates_history``
- ls → ``ls_candidates_history``
스냅샷 없으면 ``None`` (필터 생략 = 실시간 후보 유지).
**당일 스냅샷만** 사용한다. 주말 재시작·키움 매니저 다운 뒤 남은
며칠 전 history 와 sticky 후보를 교집합하면 전원 탈락(11→0) 한다.
RAM TTL(``UNIVERSE_SLOT_CACHE_TTL_SEC`` 기본 1초): 같은 슬롯키면 DB 재조회 생략.
TTL 만료 시 **single-flight**: 같은 ``(sid,src,slot_key)`` 는 DB SELECT 1회만,
동시 호출은 Event 대기 후 동일 RAM 결과 (분 단위 freeze / at_time 고정 아님).
"""
when = when or dt.now()
at_time = when.strftime("%Y-%m-%d %H:%M:%S")
sid = (strategy_id or "").upper()
src = str(history_source or "").strip().lower()
if universe_source is not None:
src = resolve_live_universe_history_source(
sid, universe_source=universe_source,
)
elif src in ("", "auto"):
src = resolve_live_universe_history_source(sid, universe_source=None)
if src not in ("ls", "kiwoom"):
src = "kiwoom"
iv = resolve_live_scan_interval_min(sid)
slot_key = slot_key_from_dt(when, iv)
cache_key = (sid, src, slot_key)
ttl = _slot_univ_ttl_sec()
day_ymd = when.strftime("%Y-%m-%d")
def _from_cache_locked(
*, as_hit: bool, from_flight: bool = False,
) -> Tuple[bool, Optional[Set[str]]]:
"""(hit, codes). hit=False 이면 미스. codes=None 은 스냅샷 없음(필터 생략)."""
hit = _SLOT_UNIV_CACHE.get(cache_key)
if hit is None:
return False, None
codes_cached, ts = hit
now_ts = time.time()
# TTL 0 이하 = 캐시 끔(스키마·ENV 설명과 동일) → 항상 미스
if ttl <= 0 or (now_ts - ts) >= ttl:
return False, None
if as_hit and prof is not None:
prof["slot_cache_hit"] = float(prof.get("slot_cache_hit", 0) or 0) + 1.0
if from_flight:
prof["slot_flight_wait"] = float(
prof.get("slot_flight_wait", 0) or 0
) + 1.0
if codes_cached is None:
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
return True, None
if prof is not None:
prof["slot_ok"] = float(prof.get("slot_ok", 0) or 0) + 1.0
prof["slot_codes"] = float(len(codes_cached))
return True, set(codes_cached)
def _store_locked(store: Optional[Set[str]]) -> None:
_SLOT_UNIV_CACHE[cache_key] = (
set(store) if store else None,
time.time(),
)
def _fetch_db() -> Optional[Set[str]]:
# 당일(캘린더) 스냅샷만 슬롯정합에 쓴다 — 낡은 history 전멸 방지
t_et = time.perf_counter()
et = _history_snapshot_event_time(db, sid, at_time, src)
_prof_add(prof, "slot_et_ms", t_et)
if not et or _event_time_ymd(et) != day_ymd:
with _SLOT_UNIV_LOCK:
_store_locked(None)
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
return None
getter = getattr(db, "get_universe_at", None)
if getter is None:
with _SLOT_UNIV_LOCK:
_store_locked(None)
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
return None
try:
t_get = time.perf_counter()
# TradeDBExt 는 history_source 지원. FakeDB 등 구시그니처는 무시.
try:
rows = getter(
strategy_id=sid, at_time=at_time, history_source=src,
) or []
except TypeError:
rows = getter(strategy_id=sid, at_time=at_time) or []
_prof_add(prof, "slot_get_ms", t_get)
except Exception:
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
return None
codes: Set[str] = set()
for r in rows or []:
code = str(r.get("code") or "").strip()
if code:
codes.add(code)
store: Optional[Set[str]] = set(codes) if codes else None
with _SLOT_UNIV_LOCK:
_store_locked(store)
if prof is not None:
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
if codes:
prof["slot_ok"] = float(prof.get("slot_ok", 0) or 0) + 1.0
prof["slot_codes"] = float(len(codes))
else:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
return codes if codes else None
# 1) TTL hit / 2) flight 대기 / 3) 리더 SELECT
leader = False
flight_ev: Optional[threading.Event] = None
with _SLOT_UNIV_LOCK:
ok, cached = _from_cache_locked(as_hit=True)
if ok:
return cached
flight_ev = _SLOT_UNIV_FLIGHT.get(cache_key)
if flight_ev is None:
flight_ev = threading.Event()
_SLOT_UNIV_FLIGHT[cache_key] = flight_ev
leader = True
if not leader and flight_ev is not None:
flight_ev.wait(timeout=max(0.05, _slot_univ_flight_wait_sec()))
with _SLOT_UNIV_LOCK:
ok, cached = _from_cache_locked(as_hit=True, from_flight=True)
if ok:
return cached
# 리더 실패·타임아웃 → 내가 재비행
if cache_key not in _SLOT_UNIV_FLIGHT:
flight_ev = threading.Event()
_SLOT_UNIV_FLIGHT[cache_key] = flight_ev
leader = True
else:
flight_ev = _SLOT_UNIV_FLIGHT[cache_key]
if not leader and flight_ev is not None:
flight_ev.wait(timeout=max(0.05, _slot_univ_flight_wait_sec()))
with _SLOT_UNIV_LOCK:
ok, cached = _from_cache_locked(as_hit=True, from_flight=True)
if ok:
return cached
if cache_key not in _SLOT_UNIV_FLIGHT:
flight_ev = threading.Event()
_SLOT_UNIV_FLIGHT[cache_key] = flight_ev
leader = True
else:
# 여전히 비행 중이면 중복 SELECT 대신 스냅샷 없음과 동일(필터 생략)
if prof is not None:
prof["slot_miss"] = float(
prof.get("slot_miss", 0) or 0
) + 1.0
return None
if not leader:
return None
assert flight_ev is not None
try:
return _fetch_db()
finally:
with _SLOT_UNIV_LOCK:
_SLOT_UNIV_FLIGHT.pop(cache_key, None)
try:
flight_ev.set()
except Exception:
pass
def filter_candidates_by_history_universe(
candidates: List[Dict],
db: Any,
strategy_id: str,
*,
when: Optional[dt] = None,
history_source: Optional[str] = None,
universe_source: Optional[str] = None,
prof: Optional[Dict[str, float]] = None,
) -> Tuple[List[Dict], int]:
"""실시간 후보 ∩ history 스냅샷. (filtered, dropped_count)."""
if not candidates:
return [], 0
hs = history_source
if hs is None:
hs = resolve_live_universe_history_source(
strategy_id, universe_source=universe_source,
)
allowed = history_universe_codes_at(
db,
strategy_id,
when,
history_source=hs or "kiwoom",
universe_source=None,
prof=prof,
)
if allowed is None:
return candidates, 0
out: List[Dict] = []
for c in candidates:
code = (c.get("code") or c.get("stk_cd") or "").strip()
if code and code in allowed:
out.append(c)
return out, len(candidates) - len(out)
def live_max_daily(strategy_id: str) -> int:
sid = (strategy_id or "").upper()
v = get_env_int(f"{sid}_MAX_DAILY", 0)
if v > 0:
return int(v)
if sid == "SHORT":
return get_env_int("TAIL_MAX_DAILY", 0)
return 0
def live_daily_entry_count(
db: Any,
code: str,
today: str,
strategy_id: str,
*,
prof: Optional[Dict[str, float]] = None,
) -> int:
try:
from .today_trades_cache import get_today_trades_cached
t0 = time.perf_counter()
trades, from_cache, db_ms = get_today_trades_cached(db, today)
if from_cache:
_prof_add(prof, "guard_trades_cache_ms", t0)
if prof is not None:
prof["guard_trades_hit"] = float(prof.get("guard_trades_hit", 0) or 0) + 1.0
else:
if prof is not None:
prof["guard_trades_db_ms"] = float(prof.get("guard_trades_db_ms", 0) or 0) + float(db_ms or 0)
prof["guard_trades_n"] = float(prof.get("guard_trades_n", 0) or 0) + 1.0
sid = (strategy_id or "").upper()
return len([
t for t in trades
if t.get("code") == code
and str(t.get("strategy", "")).upper().startswith(sid)
])
except Exception:
return 0
def resolve_live_buy_qty(
curr_price: float,
holdings: Dict[str, Dict],
strategy_id: str,
slot_money: float,
*,
max_stocks: int,
invest_cap: Optional[float] = None,
) -> Tuple[int, float, Optional[str]]:
"""
포트폴리오 백테 ``target_qty_and_cost`` + ``min_invest_ratio`` + ``total_budget`` 정합.
Returns: (qty, invest_krw, reject_reason)
"""
if curr_price <= 0:
return 0, 0.0, "가격0"
cap = float(invest_cap if invest_cap is not None else slot_money)
if cap <= 0:
return 0, 0.0, "투자캡0"
exposure = portfolio_exposure_krw(holdings)
total_budget = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money)
remaining = max(0.0, total_budget - exposure)
ratio = resolve_live_min_invest_ratio(strategy_id)
target_qty, target_cost = target_qty_and_cost(curr_price, cap)
min_required = target_cost * ratio
if target_qty < 1 or remaining < min_required:
return 0, 0.0, "소액/잔여부족"
invest = min(cap, remaining, target_cost)
qty = invest_qty_for_price(curr_price, invest)
if qty < 1:
return 0, 0.0, "수량0"
cost = qty * curr_price
if cost < min_required:
return 0, 0.0, "소액"
return qty, invest, None
def live_portfolio_entry_reject(
db: Any,
holdings: Dict[str, Dict],
strategy_id: str,
code: str,
slot_money: float,
max_stocks: int,
*,
prof: Optional[Dict[str, float]] = None,
) -> Optional[str]:
"""일일한도·총한도 — ``None`` 이면 통과."""
t0 = time.perf_counter()
enabled = live_portfolio_budget_align_enabled(strategy_id)
_prof_add(prof, "guard_env_ms", t0)
if not enabled:
return None
t1 = time.perf_counter()
md = live_max_daily(strategy_id)
_prof_add(prof, "guard_env_ms", t1)
if md > 0:
today = dt.now().strftime("%Y%m%d")
t2 = time.perf_counter()
cnt = live_daily_entry_count(db, code, today, strategy_id, prof=prof)
_prof_add(prof, "guard_daily_ms", t2)
if cnt >= md:
return f"탈락-일일한도 daily={cnt}/{md}"
t3 = time.perf_counter()
total = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money)
if total > 0 and portfolio_exposure_krw(holdings) >= total - 1e-6:
_prof_add(prof, "guard_budget_ms", t3)
return "탈락-총한도"
_prof_add(prof, "guard_budget_ms", t3)
return None
def live_portfolio_budget_full(
holdings: Dict[str, Dict],
strategy_id: str,
slot_money: float,
max_stocks: int,
) -> bool:
if not live_portfolio_budget_align_enabled(strategy_id):
return False
total = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money)
if total <= 0:
return False
return portfolio_exposure_krw(holdings) >= total - 1e-6