""" 실매 ↔ 포트폴리오 백테 정합 — 유니버스 슬롯·총한도·매수금액. 백테 ``breakout_portfolio_backtest`` / ``backtest_portfolio_common`` 과 동일 분모. """ from __future__ import annotations import threading import time from datetime import datetime as dt from typing import Any, Dict, List, Optional, Set, Tuple from ..backtest.backtest_portfolio_common import ( STRATEGY_PORTFOLIO_KEYS, min_invest_ratio_of_slot, portfolio_exposure_krw, resolve_portfolio_params, target_qty_and_cost, ) from ..utils.env import get_env_bool, get_env_float, get_env_from_db, get_env_int from ..utils.position_sizing import invest_qty_for_price # 자체 유니버스 — 기본 슬롯 필터 OFF (env 로 켤 수 있음) _UNIVERSE_SLOT_SKIP_DEFAULT = frozenset({"UPDOW", "DBBAND"}) # 총한도·일1회 base 가드 스킵 (전략 자체 구현) _PORTFOLIO_BUDGET_SKIP_DEFAULT = frozenset({"UPDOW"}) # 유니버스 슬롯 history — RAM TTL (정합 ON 유지 · 루프마다 SELECT 금지) _SLOT_UNIV_LOCK = threading.Lock() # key=(sid, src, slot_key) → (codes|None, ts). codes=None → 스냅샷 없음(필터 생략) _SLOT_UNIV_CACHE: Dict[Tuple[str, str, str], Tuple[Optional[Set[str]], float]] = {} # 같은 키 TTL 만료 시 DB SELECT 1회만 (대기자는 Event 공유) — 분슬롯 freeze 아님 _SLOT_UNIV_FLIGHT: Dict[Tuple[str, str, str], threading.Event] = {} def _slot_univ_ttl_sec() -> float: try: return float(get_env_float("UNIVERSE_SLOT_CACHE_TTL_SEC", 1.0) or 1.0) except Exception: return 1.0 def _slot_univ_flight_wait_sec() -> float: """single-flight 대기 상한(초). 리더 실패/타임아웃 후 재시도용.""" try: return float(get_env_float("UNIVERSE_SLOT_FLIGHT_WAIT_SEC", 5.0) or 5.0) except Exception: return 5.0 def invalidate_universe_slot_cache(strategy_id: Optional[str] = None) -> None: """슬롯 캐시 무효화. strategy_id 없으면 전체. in-flight 대기자도 깨움.""" sid = (strategy_id or "").upper() wake: List[threading.Event] = [] with _SLOT_UNIV_LOCK: if not sid: _SLOT_UNIV_CACHE.clear() wake = list(_SLOT_UNIV_FLIGHT.values()) _SLOT_UNIV_FLIGHT.clear() else: for k in list(_SLOT_UNIV_CACHE.keys()): if k[0] == sid: del _SLOT_UNIV_CACHE[k] for k in list(_SLOT_UNIV_FLIGHT.keys()): if k[0] == sid: wake.append(_SLOT_UNIV_FLIGHT.pop(k)) for ev in wake: try: ev.set() except Exception: pass def _prof_add(prof: Optional[Dict[str, float]], key: str, t0: float) -> None: """LOOP_PROF 세분용 — 매매 로직 불변, ms 누적만.""" if prof is None: return try: prof[key] = float(prof.get(key, 0.0) or 0.0) + (time.perf_counter() - t0) * 1000.0 except Exception: pass def portfolio_strategy_key(strategy_id: str) -> str: sid = (strategy_id or "").upper() return "TAIL" if sid == "SHORT" else sid def _env_flag(key: str, default: bool) -> bool: raw = get_env_from_db(key, "") if raw in (None, "", "None"): return default return str(raw).strip().lower() in ("1", "true", "t", "y", "yes", "on") def live_universe_slot_align_enabled(strategy_id: str) -> bool: """실매 후보 ∩ history 슬롯 필터. 기본 **False** — history 지연/축소 시 전량탈락(universe_wipe) 방지. 백테 슬롯 정합이 필요하면 ``LIVE_UNIVERSE_SLOT_ALIGN=true`` 또는 전략별 키로 ON. """ sid = (strategy_id or "").upper() per_key = f"{sid}_LIVE_UNIVERSE_SLOT_ALIGN" if get_env_from_db(per_key, "") not in (None, "", "None"): return _env_flag(per_key, False) if sid in _UNIVERSE_SLOT_SKIP_DEFAULT: return False return _env_flag("LIVE_UNIVERSE_SLOT_ALIGN", False) def resolve_live_universe_history_source( strategy_id: str, *, universe_source: Optional[str] = None, ) -> str: """실매 슬롯 정합이 읽을 이력 테이블. - ``ls_condition`` 유니버스 → ``ls_candidates_history`` - 그 외 → ``target_candidates_history`` (키움/KIS) 키움 이력과 LS 실후보를 교집합하면 전원 탈락(20→0) 한다. """ sid = (strategy_id or "").upper() # 전략·전역 명시 오버라이드 for key in ( f"{sid}_LIVE_UNIVERSE_HISTORY_SOURCE", "LIVE_UNIVERSE_HISTORY_SOURCE", ): raw = get_env_from_db(key, "") if raw not in (None, "", "None"): s = str(raw).strip().lower() if s in ("ls", "ls_condition", "ls_afr"): return "ls" if s in ("kiwoom", "target", "kis", "condition"): return "kiwoom" src = str(universe_source or "").strip().lower() if not src: src = str( get_env_from_db(f"{sid}_UNIVERSE_SOURCE", "") or "" ).strip().lower() if src in ("ls_condition", "ls", "ls_afr"): return "ls" return "kiwoom" def live_portfolio_budget_align_enabled(strategy_id: str) -> bool: sid = (strategy_id or "").upper() per_key = f"{sid}_LIVE_PORTFOLIO_BUDGET_ALIGN" if get_env_from_db(per_key, "") not in (None, "", "None"): return _env_flag(per_key, True) if sid in _PORTFOLIO_BUDGET_SKIP_DEFAULT: return False return _env_flag("LIVE_PORTFOLIO_BUDGET_ALIGN", True) def resolve_live_scan_interval_min(strategy_id: str) -> int: sid = (strategy_id or "").upper() v = get_env_int(f"{sid}_SCAN_INTERVAL_MIN", 0) if v > 0: return max(1, int(v)) return max(1, get_env_int("SCAN_INTERVAL_MIN", 1)) def slot_key_from_dt(when: dt, scan_interval_min: int = 1) -> str: """백테 ``_bt_slot_key`` / ``_slot_key`` 와 동일.""" iv = max(1, int(scan_interval_min)) date = when.strftime("%Y%m%d") hm = when.hour * 100 + when.minute total_min = (hm // 100) * 60 + (hm % 100) slot_min = (total_min // iv) * iv slot_hm = (slot_min // 60) * 100 + (slot_min % 60) return date + str(slot_hm).zfill(4) def resolve_live_total_budget_krw( strategy_id: str, max_stocks: int, slot_money: float, ) -> float: pkey = portfolio_strategy_key(strategy_id) keys = STRATEGY_PORTFOLIO_KEYS.get(pkey, STRATEGY_PORTFOLIO_KEYS["SCALP"]) env_row: Dict[str, Any] = {} for k in keys.get("total_budget", ()): v = get_env_int(k, 0) if v > 0: env_row[k] = v break for k in keys.get("slot", ()): v = get_env_int(k, 0) if v > 0: env_row[k] = v for k in keys.get("max_stocks", ()): v = get_env_int(k, 0) if v > 0: env_row[k] = v pf = resolve_portfolio_params( env_row if env_row else None, strategy=pkey, slot_money=float(slot_money), max_stocks=int(max_stocks), ) return float(pf["total_budget_krw"]) def resolve_live_min_invest_ratio(strategy_id: str) -> float: return min_invest_ratio_of_slot({}, strategy=portfolio_strategy_key(strategy_id)) def _history_snapshot_event_time( db: Any, strategy_id: str, at_time: str, history_source: str, ) -> Optional[Any]: """``at_time`` 이전 최신 ``event_time`` (없으면 None).""" conn = getattr(db, "conn", None) if conn is None: return None table_fn = getattr(db, "_universe_history_table", None) if callable(table_fn): try: table = table_fn(history_source) except Exception: table = None else: table = None if not table: from kis_trader.backtest.universe_history_source import history_table_for_source table = history_table_for_source(history_source) try: row = conn.execute( f""" SELECT MAX(event_time) AS et FROM {table} WHERE strategy_id=%s AND event_time <= %s """, (strategy_id, at_time), ).fetchone() return (row or {}).get("et") if row else None except Exception: return None def _event_time_ymd(et: Any) -> str: if et is None: return "" if hasattr(et, "strftime"): try: return et.strftime("%Y-%m-%d") except Exception: pass s = str(et).strip() if len(s) >= 10 and s[4] == "-" and s[7] == "-": return s[:10] # YYYYMMDDHHMM / YYYYMMDD... digits = "".join(ch for ch in s if ch.isdigit()) if len(digits) >= 8: return f"{digits[:4]}-{digits[4:6]}-{digits[6:8]}" return "" def history_universe_codes_at( db: Any, strategy_id: str, when: Optional[dt] = None, *, history_source: str = "kiwoom", universe_source: Optional[str] = None, prof: Optional[Dict[str, float]] = None, ) -> Optional[Set[str]]: """ 이력 스냅샷 — ``at_time`` 이전 최신 event_time 종목 집합. - kiwoom → ``target_candidates_history`` - ls → ``ls_candidates_history`` 스냅샷 없으면 ``None`` (필터 생략 = 실시간 후보 유지). **당일 스냅샷만** 사용한다. 주말 재시작·키움 매니저 다운 뒤 남은 며칠 전 history 와 sticky 후보를 교집합하면 전원 탈락(11→0) 한다. RAM TTL(``UNIVERSE_SLOT_CACHE_TTL_SEC`` 기본 1초): 같은 슬롯키면 DB 재조회 생략. TTL 만료 시 **single-flight**: 같은 ``(sid,src,slot_key)`` 는 DB SELECT 1회만, 동시 호출은 Event 대기 후 동일 RAM 결과 (분 단위 freeze / at_time 고정 아님). """ when = when or dt.now() at_time = when.strftime("%Y-%m-%d %H:%M:%S") sid = (strategy_id or "").upper() src = str(history_source or "").strip().lower() if universe_source is not None: src = resolve_live_universe_history_source( sid, universe_source=universe_source, ) elif src in ("", "auto"): src = resolve_live_universe_history_source(sid, universe_source=None) if src not in ("ls", "kiwoom"): src = "kiwoom" iv = resolve_live_scan_interval_min(sid) slot_key = slot_key_from_dt(when, iv) cache_key = (sid, src, slot_key) ttl = _slot_univ_ttl_sec() day_ymd = when.strftime("%Y-%m-%d") def _from_cache_locked( *, as_hit: bool, from_flight: bool = False, ) -> Tuple[bool, Optional[Set[str]]]: """(hit, codes). hit=False 이면 미스. codes=None 은 스냅샷 없음(필터 생략).""" hit = _SLOT_UNIV_CACHE.get(cache_key) if hit is None: return False, None codes_cached, ts = hit now_ts = time.time() # TTL 0 이하 = 캐시 끔(스키마·ENV 설명과 동일) → 항상 미스 if ttl <= 0 or (now_ts - ts) >= ttl: return False, None if as_hit and prof is not None: prof["slot_cache_hit"] = float(prof.get("slot_cache_hit", 0) or 0) + 1.0 if from_flight: prof["slot_flight_wait"] = float( prof.get("slot_flight_wait", 0) or 0 ) + 1.0 if codes_cached is None: if prof is not None: prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0 return True, None if prof is not None: prof["slot_ok"] = float(prof.get("slot_ok", 0) or 0) + 1.0 prof["slot_codes"] = float(len(codes_cached)) return True, set(codes_cached) def _store_locked(store: Optional[Set[str]]) -> None: _SLOT_UNIV_CACHE[cache_key] = ( set(store) if store else None, time.time(), ) def _fetch_db() -> Optional[Set[str]]: # 당일(캘린더) 스냅샷만 슬롯정합에 쓴다 — 낡은 history 전멸 방지 t_et = time.perf_counter() et = _history_snapshot_event_time(db, sid, at_time, src) _prof_add(prof, "slot_et_ms", t_et) if not et or _event_time_ymd(et) != day_ymd: with _SLOT_UNIV_LOCK: _store_locked(None) if prof is not None: prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0 prof["slot_cache_miss"] = float( prof.get("slot_cache_miss", 0) or 0 ) + 1.0 return None getter = getattr(db, "get_universe_at", None) if getter is None: with _SLOT_UNIV_LOCK: _store_locked(None) if prof is not None: prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0 prof["slot_cache_miss"] = float( prof.get("slot_cache_miss", 0) or 0 ) + 1.0 return None try: t_get = time.perf_counter() # TradeDBExt 는 history_source 지원. FakeDB 등 구시그니처는 무시. try: rows = getter( strategy_id=sid, at_time=at_time, history_source=src, ) or [] except TypeError: rows = getter(strategy_id=sid, at_time=at_time) or [] _prof_add(prof, "slot_get_ms", t_get) except Exception: if prof is not None: prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0 prof["slot_cache_miss"] = float( prof.get("slot_cache_miss", 0) or 0 ) + 1.0 return None codes: Set[str] = set() for r in rows or []: code = str(r.get("code") or "").strip() if code: codes.add(code) store: Optional[Set[str]] = set(codes) if codes else None with _SLOT_UNIV_LOCK: _store_locked(store) if prof is not None: prof["slot_cache_miss"] = float( prof.get("slot_cache_miss", 0) or 0 ) + 1.0 if codes: prof["slot_ok"] = float(prof.get("slot_ok", 0) or 0) + 1.0 prof["slot_codes"] = float(len(codes)) else: prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0 return codes if codes else None # 1) TTL hit / 2) flight 대기 / 3) 리더 SELECT leader = False flight_ev: Optional[threading.Event] = None with _SLOT_UNIV_LOCK: ok, cached = _from_cache_locked(as_hit=True) if ok: return cached flight_ev = _SLOT_UNIV_FLIGHT.get(cache_key) if flight_ev is None: flight_ev = threading.Event() _SLOT_UNIV_FLIGHT[cache_key] = flight_ev leader = True if not leader and flight_ev is not None: flight_ev.wait(timeout=max(0.05, _slot_univ_flight_wait_sec())) with _SLOT_UNIV_LOCK: ok, cached = _from_cache_locked(as_hit=True, from_flight=True) if ok: return cached # 리더 실패·타임아웃 → 내가 재비행 if cache_key not in _SLOT_UNIV_FLIGHT: flight_ev = threading.Event() _SLOT_UNIV_FLIGHT[cache_key] = flight_ev leader = True else: flight_ev = _SLOT_UNIV_FLIGHT[cache_key] if not leader and flight_ev is not None: flight_ev.wait(timeout=max(0.05, _slot_univ_flight_wait_sec())) with _SLOT_UNIV_LOCK: ok, cached = _from_cache_locked(as_hit=True, from_flight=True) if ok: return cached if cache_key not in _SLOT_UNIV_FLIGHT: flight_ev = threading.Event() _SLOT_UNIV_FLIGHT[cache_key] = flight_ev leader = True else: # 여전히 비행 중이면 중복 SELECT 대신 스냅샷 없음과 동일(필터 생략) if prof is not None: prof["slot_miss"] = float( prof.get("slot_miss", 0) or 0 ) + 1.0 return None if not leader: return None assert flight_ev is not None try: return _fetch_db() finally: with _SLOT_UNIV_LOCK: _SLOT_UNIV_FLIGHT.pop(cache_key, None) try: flight_ev.set() except Exception: pass def filter_candidates_by_history_universe( candidates: List[Dict], db: Any, strategy_id: str, *, when: Optional[dt] = None, history_source: Optional[str] = None, universe_source: Optional[str] = None, prof: Optional[Dict[str, float]] = None, ) -> Tuple[List[Dict], int]: """실시간 후보 ∩ history 스냅샷. (filtered, dropped_count).""" if not candidates: return [], 0 hs = history_source if hs is None: hs = resolve_live_universe_history_source( strategy_id, universe_source=universe_source, ) allowed = history_universe_codes_at( db, strategy_id, when, history_source=hs or "kiwoom", universe_source=None, prof=prof, ) if allowed is None: return candidates, 0 out: List[Dict] = [] for c in candidates: code = (c.get("code") or c.get("stk_cd") or "").strip() if code and code in allowed: out.append(c) return out, len(candidates) - len(out) def live_max_daily(strategy_id: str) -> int: sid = (strategy_id or "").upper() v = get_env_int(f"{sid}_MAX_DAILY", 0) if v > 0: return int(v) if sid == "SHORT": return get_env_int("TAIL_MAX_DAILY", 0) return 0 def live_daily_entry_count( db: Any, code: str, today: str, strategy_id: str, *, prof: Optional[Dict[str, float]] = None, ) -> int: try: from .today_trades_cache import get_today_trades_cached t0 = time.perf_counter() trades, from_cache, db_ms = get_today_trades_cached(db, today) if from_cache: _prof_add(prof, "guard_trades_cache_ms", t0) if prof is not None: prof["guard_trades_hit"] = float(prof.get("guard_trades_hit", 0) or 0) + 1.0 else: if prof is not None: prof["guard_trades_db_ms"] = float(prof.get("guard_trades_db_ms", 0) or 0) + float(db_ms or 0) prof["guard_trades_n"] = float(prof.get("guard_trades_n", 0) or 0) + 1.0 sid = (strategy_id or "").upper() return len([ t for t in trades if t.get("code") == code and str(t.get("strategy", "")).upper().startswith(sid) ]) except Exception: return 0 def resolve_live_buy_qty( curr_price: float, holdings: Dict[str, Dict], strategy_id: str, slot_money: float, *, max_stocks: int, invest_cap: Optional[float] = None, ) -> Tuple[int, float, Optional[str]]: """ 포트폴리오 백테 ``target_qty_and_cost`` + ``min_invest_ratio`` + ``total_budget`` 정합. Returns: (qty, invest_krw, reject_reason) """ if curr_price <= 0: return 0, 0.0, "가격0" cap = float(invest_cap if invest_cap is not None else slot_money) if cap <= 0: return 0, 0.0, "투자캡0" exposure = portfolio_exposure_krw(holdings) total_budget = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money) remaining = max(0.0, total_budget - exposure) ratio = resolve_live_min_invest_ratio(strategy_id) target_qty, target_cost = target_qty_and_cost(curr_price, cap) min_required = target_cost * ratio if target_qty < 1 or remaining < min_required: return 0, 0.0, "소액/잔여부족" invest = min(cap, remaining, target_cost) qty = invest_qty_for_price(curr_price, invest) if qty < 1: return 0, 0.0, "수량0" cost = qty * curr_price if cost < min_required: return 0, 0.0, "소액" return qty, invest, None def live_portfolio_entry_reject( db: Any, holdings: Dict[str, Dict], strategy_id: str, code: str, slot_money: float, max_stocks: int, *, prof: Optional[Dict[str, float]] = None, ) -> Optional[str]: """일일한도·총한도 — ``None`` 이면 통과.""" t0 = time.perf_counter() enabled = live_portfolio_budget_align_enabled(strategy_id) _prof_add(prof, "guard_env_ms", t0) if not enabled: return None t1 = time.perf_counter() md = live_max_daily(strategy_id) _prof_add(prof, "guard_env_ms", t1) if md > 0: today = dt.now().strftime("%Y%m%d") t2 = time.perf_counter() cnt = live_daily_entry_count(db, code, today, strategy_id, prof=prof) _prof_add(prof, "guard_daily_ms", t2) if cnt >= md: return f"탈락-일일한도 daily={cnt}/{md}" t3 = time.perf_counter() total = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money) if total > 0 and portfolio_exposure_krw(holdings) >= total - 1e-6: _prof_add(prof, "guard_budget_ms", t3) return "탈락-총한도" _prof_add(prof, "guard_budget_ms", t3) return None def live_portfolio_budget_full( holdings: Dict[str, Dict], strategy_id: str, slot_money: float, max_stocks: int, ) -> bool: if not live_portfolio_budget_align_enabled(strategy_id): return False total = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money) if total <= 0: return False return portfolio_exposure_krw(holdings) >= total - 1e-6