hoon/kis_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

거래 빠르게 안티에서 병신만든거 커서로

Browse Source 
feat: Implement backtest source management and enhance candle data handling

Changes:
- Introduced a new function `_apply_backtest_source_env_from_request` to manage the environment variables for candle, tick, and order book sources based on incoming requests.
- Added a teardown function `_teardown_backtest_source_env` to ensure that environment variables do not persist between requests, enhancing the stability of the backtesting environment.
- Refactored existing code to utilize the new source management functions, improving code readability and maintainability.
- Added new utility functions in `bt_candle_source.py` for fetching and managing candle data, ensuring consistency with live trading data sources.

Impact:
- These changes improve the flexibility and reliability of the backtesting framework, allowing for better management of data sources and reducing the risk of cross-request contamination.
This commit is contained in:
Your Name
2026-08-13 16:03:40 +09:00
parent c6bd62a25f
commit 2c7ad867f4
53 changed files with 15251 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
kis_trader/utils/env.py
View File

@@ -19,6 +19,9 @@ _db_instance = None
# get_merged_env_snapshot() — TTL 내 재사용 (실매: 웹에서 MAX_DAILY 등 변경 즉시 반영)
_merged_env_cache: Optional[Dict[str, str]] = None
_merged_env_cache_ts: float = 0.0
# get_strategy_env_dict — 전략 config_* 스냅샷 RAM (매 루프 DB 금지)
# 유니버스/봉 정합과 무관. 임계값·한도만. 기본 1초 · 분단위 동결 금지.
_strategy_env_cache: Dict[str, Any] = {}
# env 캐시 세대 카운터 — invalidate 시 +1. 파생 캐시(예: whipsaw 파라미터)가
# 이 값으로 무효화를 감지해 안전하게 재계산한다(값은 그대로, 재계산 시점만 동일).
_env_generation: int = 0
@@ -45,9 +48,10 @@ def set_db(db_obj) -> None:
def invalidate_merged_env_cache() -> None:
"""env/config 저장 후 스냅샷 캐시 무효화 (insert_env_snapshot 등)."""
global _merged_env_cache, _merged_env_cache_ts, _env_generation
global _merged_env_cache, _merged_env_cache_ts, _env_generation, _strategy_env_cache
_merged_env_cache = None
_merged_env_cache_ts = 0.0
_strategy_env_cache = {}
_env_generation += 1
@@ -59,6 +63,17 @@ def _merged_env_cache_ttl_sec() -> float:
return 60.0
def _strategy_env_cache_ttl_sec() -> float:
"""전략 config_* RAM TTL(초). 기본 1 — 유니버스 분슬롯과 무관.
웹 저장 시 invalidate_merged_env_cache 로 즉시 무효화.
"""
try:
return max(0.0, float(os.environ.get("STRATEGY_ENV_CACHE_TTL_SEC", "1")))
except (ValueError, TypeError):
return 1.0
def env_cache_generation() -> int:
"""현재 env 캐시 세대. invalidate 될 때마다 증가.
@@ -105,21 +120,38 @@ def get_strategy_env_dict(strategy_id: str) -> dict:
SCALP → config_scalp, MOMENTUM → config_momentum, SHORT → config_short …
실매(get_env_from_db) · 웹 · 파라서치가 동일 소스를 쓰도록 한다.
RAM TTL(``STRATEGY_ENV_CACHE_TTL_SEC`` 기본 1초): 매 루프 config_* SELECT 금지.
유니버스 event_time/분슬롯과 무관 — 손절·한도 등 **설정값**만.
웹 저장 → ``invalidate_merged_env_cache`` 즉시 반영.
"""
global _strategy_env_cache
sid = str(strategy_id or "").strip().upper() or "_"
ttl = _strategy_env_cache_ttl_sec()
now = time.time()
gen = _env_generation
hit = _strategy_env_cache.get(sid)
if (
isinstance(hit, dict)
and hit.get("gen") == gen
and (ttl <= 0 or (now - float(hit.get("ts") or 0)) < ttl)
and isinstance(hit.get("data"), dict)
):
return hit["data"]
merged = get_merged_env_dict()
db = _get_db()
if db is None:
return merged
try:
if hasattr(db, "get_strategy_config_snapshot"):
strat = db.get_strategy_config_snapshot(strategy_id)
if strat:
out = dict(merged)
out.update(strat)
return out
except Exception as e:
logger.debug("strategy env 조회 실패 (%s): %s", strategy_id, e)
return merged
out = dict(merged)
if db is not None:
try:
if hasattr(db, "get_strategy_config_snapshot"):
strat = db.get_strategy_config_snapshot(strategy_id)
if strat:
out.update(strat)
except Exception as e:
logger.debug("strategy env 조회 실패 (%s): %s", strategy_id, e)
_strategy_env_cache[sid] = {"data": out, "ts": now, "gen": gen}
return out
def get_env_from_db(key: str, default: str = "") -> str:

273
kis_trader/utils/live_portfolio_common.py
View File

@@ -4,6 +4,8 @@
"""
from __future__ import annotations
import threading
import time
from datetime import datetime as dt
from typing import Any, Dict, List, Optional, Set, Tuple
@@ -14,7 +16,7 @@ from ..backtest.backtest_portfolio_common import (
resolve_portfolio_params,
target_qty_and_cost,
)
from ..utils.env import get_env_bool, get_env_from_db, get_env_int
from ..utils.env import get_env_bool, get_env_float, get_env_from_db, get_env_int
from ..utils.position_sizing import invest_qty_for_price
# 자체 유니버스 — 기본 슬롯 필터 OFF (env 로 켤 수 있음)
@@ -23,6 +25,61 @@ _UNIVERSE_SLOT_SKIP_DEFAULT = frozenset({"UPDOW", "DBBAND"})
# 총한도·일1회 base 가드 스킵 (전략 자체 구현)
_PORTFOLIO_BUDGET_SKIP_DEFAULT = frozenset({"UPDOW"})
# 유니버스 슬롯 history — RAM TTL (정합 ON 유지 · 루프마다 SELECT 금지)
_SLOT_UNIV_LOCK = threading.Lock()
# key=(sid, src, slot_key) → (codes|None, ts). codes=None → 스냅샷 없음(필터 생략)
_SLOT_UNIV_CACHE: Dict[Tuple[str, str, str], Tuple[Optional[Set[str]], float]] = {}
# 같은 키 TTL 만료 시 DB SELECT 1회만 (대기자는 Event 공유) — 분슬롯 freeze 아님
_SLOT_UNIV_FLIGHT: Dict[Tuple[str, str, str], threading.Event] = {}
def _slot_univ_ttl_sec() -> float:
try:
return float(get_env_float("UNIVERSE_SLOT_CACHE_TTL_SEC", 1.0) or 1.0)
except Exception:
return 1.0
def _slot_univ_flight_wait_sec() -> float:
"""single-flight 대기 상한(초). 리더 실패/타임아웃 후 재시도용."""
try:
return float(get_env_float("UNIVERSE_SLOT_FLIGHT_WAIT_SEC", 5.0) or 5.0)
except Exception:
return 5.0
def invalidate_universe_slot_cache(strategy_id: Optional[str] = None) -> None:
"""슬롯 캐시 무효화. strategy_id 없으면 전체. in-flight 대기자도 깨움."""
sid = (strategy_id or "").upper()
wake: List[threading.Event] = []
with _SLOT_UNIV_LOCK:
if not sid:
_SLOT_UNIV_CACHE.clear()
wake = list(_SLOT_UNIV_FLIGHT.values())
_SLOT_UNIV_FLIGHT.clear()
else:
for k in list(_SLOT_UNIV_CACHE.keys()):
if k[0] == sid:
del _SLOT_UNIV_CACHE[k]
for k in list(_SLOT_UNIV_FLIGHT.keys()):
if k[0] == sid:
wake.append(_SLOT_UNIV_FLIGHT.pop(k))
for ev in wake:
try:
ev.set()
except Exception:
pass
def _prof_add(prof: Optional[Dict[str, float]], key: str, t0: float) -> None:
"""LOOP_PROF 세분용 — 매매 로직 불변, ms 누적만."""
if prof is None:
return
try:
prof[key] = float(prof.get(key, 0.0) or 0.0) + (time.perf_counter() - t0) * 1000.0
except Exception:
pass
def portfolio_strategy_key(strategy_id: str) -> str:
sid = (strategy_id or "").upper()
@@ -209,6 +266,7 @@ def history_universe_codes_at(
*,
history_source: str = "kiwoom",
universe_source: Optional[str] = None,
prof: Optional[Dict[str, float]] = None,
) -> Optional[Set[str]]:
"""
이력 스냅샷 — ``at_time`` 이전 최신 event_time 종목 집합.
@@ -218,6 +276,10 @@ def history_universe_codes_at(
**당일 스냅샷만** 사용한다. 주말 재시작·키움 매니저 다운 뒤 남은
며칠 전 history 와 sticky 후보를 교집합하면 전원 탈락(11→0) 한다.
RAM TTL(``UNIVERSE_SLOT_CACHE_TTL_SEC`` 기본 1초): 같은 슬롯키면 DB 재조회 생략.
TTL 만료 시 **single-flight**: 같은 ``(sid,src,slot_key)`` 는 DB SELECT 1회만,
동시 호출은 Event 대기 후 동일 RAM 결과 (분 단위 freeze / at_time 고정 아님).
"""
when = when or dt.now()
at_time = when.strftime("%Y-%m-%d %H:%M:%S")
@@ -232,34 +294,162 @@ def history_universe_codes_at(
if src not in ("ls", "kiwoom"):
src = "kiwoom"
# 당일(캘린더) 스냅샷만 슬롯정합에 쓴다 — 낡은 history 전멸 방지
et = _history_snapshot_event_time(db, sid, at_time, src)
if not et:
return None
if _event_time_ymd(et) != when.strftime("%Y-%m-%d"):
iv = resolve_live_scan_interval_min(sid)
slot_key = slot_key_from_dt(when, iv)
cache_key = (sid, src, slot_key)
ttl = _slot_univ_ttl_sec()
day_ymd = when.strftime("%Y-%m-%d")
def _from_cache_locked(
*, as_hit: bool, from_flight: bool = False,
) -> Tuple[bool, Optional[Set[str]]]:
"""(hit, codes). hit=False 이면 미스. codes=None 은 스냅샷 없음(필터 생략)."""
hit = _SLOT_UNIV_CACHE.get(cache_key)
if hit is None:
return False, None
codes_cached, ts = hit
now_ts = time.time()
# TTL 0 이하 = 캐시 끔(스키마·ENV 설명과 동일) → 항상 미스
if ttl <= 0 or (now_ts - ts) >= ttl:
return False, None
if as_hit and prof is not None:
prof["slot_cache_hit"] = float(prof.get("slot_cache_hit", 0) or 0) + 1.0
if from_flight:
prof["slot_flight_wait"] = float(
prof.get("slot_flight_wait", 0) or 0
) + 1.0
if codes_cached is None:
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
return True, None
if prof is not None:
prof["slot_ok"] = float(prof.get("slot_ok", 0) or 0) + 1.0
prof["slot_codes"] = float(len(codes_cached))
return True, set(codes_cached)
def _store_locked(store: Optional[Set[str]]) -> None:
_SLOT_UNIV_CACHE[cache_key] = (
set(store) if store else None,
time.time(),
)
def _fetch_db() -> Optional[Set[str]]:
# 당일(캘린더) 스냅샷만 슬롯정합에 쓴다 — 낡은 history 전멸 방지
t_et = time.perf_counter()
et = _history_snapshot_event_time(db, sid, at_time, src)
_prof_add(prof, "slot_et_ms", t_et)
if not et or _event_time_ymd(et) != day_ymd:
with _SLOT_UNIV_LOCK:
_store_locked(None)
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
return None
getter = getattr(db, "get_universe_at", None)
if getter is None:
with _SLOT_UNIV_LOCK:
_store_locked(None)
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
return None
try:
t_get = time.perf_counter()
# TradeDBExt 는 history_source 지원. FakeDB 등 구시그니처는 무시.
try:
rows = getter(
strategy_id=sid, at_time=at_time, history_source=src,
) or []
except TypeError:
rows = getter(strategy_id=sid, at_time=at_time) or []
_prof_add(prof, "slot_get_ms", t_get)
except Exception:
if prof is not None:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
return None
codes: Set[str] = set()
for r in rows or []:
code = str(r.get("code") or "").strip()
if code:
codes.add(code)
store: Optional[Set[str]] = set(codes) if codes else None
with _SLOT_UNIV_LOCK:
_store_locked(store)
if prof is not None:
prof["slot_cache_miss"] = float(
prof.get("slot_cache_miss", 0) or 0
) + 1.0
if codes:
prof["slot_ok"] = float(prof.get("slot_ok", 0) or 0) + 1.0
prof["slot_codes"] = float(len(codes))
else:
prof["slot_miss"] = float(prof.get("slot_miss", 0) or 0) + 1.0
return codes if codes else None
# 1) TTL hit / 2) flight 대기 / 3) 리더 SELECT
leader = False
flight_ev: Optional[threading.Event] = None
with _SLOT_UNIV_LOCK:
ok, cached = _from_cache_locked(as_hit=True)
if ok:
return cached
flight_ev = _SLOT_UNIV_FLIGHT.get(cache_key)
if flight_ev is None:
flight_ev = threading.Event()
_SLOT_UNIV_FLIGHT[cache_key] = flight_ev
leader = True
if not leader and flight_ev is not None:
flight_ev.wait(timeout=max(0.05, _slot_univ_flight_wait_sec()))
with _SLOT_UNIV_LOCK:
ok, cached = _from_cache_locked(as_hit=True, from_flight=True)
if ok:
return cached
# 리더 실패·타임아웃 → 내가 재비행
if cache_key not in _SLOT_UNIV_FLIGHT:
flight_ev = threading.Event()
_SLOT_UNIV_FLIGHT[cache_key] = flight_ev
leader = True
else:
flight_ev = _SLOT_UNIV_FLIGHT[cache_key]
if not leader and flight_ev is not None:
flight_ev.wait(timeout=max(0.05, _slot_univ_flight_wait_sec()))
with _SLOT_UNIV_LOCK:
ok, cached = _from_cache_locked(as_hit=True, from_flight=True)
if ok:
return cached
if cache_key not in _SLOT_UNIV_FLIGHT:
flight_ev = threading.Event()
_SLOT_UNIV_FLIGHT[cache_key] = flight_ev
leader = True
else:
# 여전히 비행 중이면 중복 SELECT 대신 스냅샷 없음과 동일(필터 생략)
if prof is not None:
prof["slot_miss"] = float(
prof.get("slot_miss", 0) or 0
) + 1.0
return None
if not leader:
return None
getter = getattr(db, "get_universe_at", None)
if getter is None:
return None
assert flight_ev is not None
try:
# TradeDBExt 는 history_source 지원. FakeDB 등 구시그니처는 무시.
return _fetch_db()
finally:
with _SLOT_UNIV_LOCK:
_SLOT_UNIV_FLIGHT.pop(cache_key, None)
try:
rows = getter(
strategy_id=sid, at_time=at_time, history_source=src,
) or []
except TypeError:
rows = getter(strategy_id=sid, at_time=at_time) or []
except Exception:
return None
if not rows:
return None
codes: Set[str] = set()
for r in rows:
code = str(r.get("code") or "").strip()
if code:
codes.add(code)
return codes if codes else None
flight_ev.set()
except Exception:
pass
def filter_candidates_by_history_universe(
@@ -270,6 +460,7 @@ def filter_candidates_by_history_universe(
when: Optional[dt] = None,
history_source: Optional[str] = None,
universe_source: Optional[str] = None,
prof: Optional[Dict[str, float]] = None,
) -> Tuple[List[Dict], int]:
"""실시간 후보 ∩ history 스냅샷. (filtered, dropped_count)."""
if not candidates:
@@ -285,6 +476,7 @@ def filter_candidates_by_history_universe(
when,
history_source=hs or "kiwoom",
universe_source=None,
prof=prof,
)
if allowed is None:
return candidates, 0
@@ -311,9 +503,22 @@ def live_daily_entry_count(
code: str,
today: str,
strategy_id: str,
*,
prof: Optional[Dict[str, float]] = None,
) -> int:
try:
trades = db.get_trades_by_date(today) or []
from .today_trades_cache import get_today_trades_cached
t0 = time.perf_counter()
trades, from_cache, db_ms = get_today_trades_cached(db, today)
if from_cache:
_prof_add(prof, "guard_trades_cache_ms", t0)
if prof is not None:
prof["guard_trades_hit"] = float(prof.get("guard_trades_hit", 0) or 0) + 1.0
else:
if prof is not None:
prof["guard_trades_db_ms"] = float(prof.get("guard_trades_db_ms", 0) or 0) + float(db_ms or 0)
prof["guard_trades_n"] = float(prof.get("guard_trades_n", 0) or 0) + 1.0
sid = (strategy_id or "").upper()
return len([
t for t in trades
@@ -367,19 +572,31 @@ def live_portfolio_entry_reject(
code: str,
slot_money: float,
max_stocks: int,
*,
prof: Optional[Dict[str, float]] = None,
) -> Optional[str]:
"""일일한도·총한도 — ``None`` 이면 통과."""
if not live_portfolio_budget_align_enabled(strategy_id):
t0 = time.perf_counter()
enabled = live_portfolio_budget_align_enabled(strategy_id)
_prof_add(prof, "guard_env_ms", t0)
if not enabled:
return None
t1 = time.perf_counter()
md = live_max_daily(strategy_id)
_prof_add(prof, "guard_env_ms", t1)
if md > 0:
today = dt.now().strftime("%Y%m%d")
cnt = live_daily_entry_count(db, code, today, strategy_id)
t2 = time.perf_counter()
cnt = live_daily_entry_count(db, code, today, strategy_id, prof=prof)
_prof_add(prof, "guard_daily_ms", t2)
if cnt >= md:
return f"탈락-일일한도 daily={cnt}/{md}"
t3 = time.perf_counter()
total = resolve_live_total_budget_krw(strategy_id, max_stocks, slot_money)
if total > 0 and portfolio_exposure_krw(holdings) >= total - 1e-6:
_prof_add(prof, "guard_budget_ms", t3)
return "탈락-총한도"
_prof_add(prof, "guard_budget_ms", t3)
return None

64
kis_trader/utils/today_trades_cache.py Normal file
View File

@@ -0,0 +1,64 @@
"""당일 trade_history — 프로세스 공유 RAM + TTL.
check_buy·포트가드가 같은 표를 나눠 씀. SELECT 연타·공유 DB 락 대기 완화.
TTL 만료 또는 invalidate 시에만 DB 재조회 (DB 쓰기 아님).
"""
from __future__ import annotations
import threading
import time
from typing import Any, Dict, List, Optional, Tuple
_LOCK = threading.Lock()
_DAY: str = ""
_ROWS: List[Dict] = []
_TS: float = 0.0
def _ttl_sec() -> float:
try:
from .env import get_env_float
return float(get_env_float("TODAY_TRADES_CACHE_TTL_SEC", 1.0) or 1.0)
except Exception:
return 1.0
def invalidate_today_trades_cache() -> None:
"""매수 체결 직후 등 — 다음 get 이 DB 재조회."""
global _DAY, _ROWS, _TS
with _LOCK:
_DAY = ""
_ROWS = []
_TS = 0.0
def get_today_trades_cached(
db: Any,
today: Optional[str] = None,
) -> Tuple[List[Dict], bool, float]:
"""Returns: (rows, from_cache, db_ms).
from_cache=True 이면 DB 미호출.
"""
global _DAY, _ROWS, _TS
from datetime import datetime as dt
day = str(today or dt.now().strftime("%Y%m%d"))
ttl = _ttl_sec()
now = time.time()
with _LOCK:
if _DAY == day and (ttl <= 0 or (now - _TS) < ttl):
return list(_ROWS), True, 0.0
t0 = time.perf_counter()
try:
rows = list(db.get_trades_by_date(day) or [])
except Exception:
rows = []
db_ms = (time.perf_counter() - t0) * 1000.0
with _LOCK:
_DAY = day
_ROWS = rows
_TS = time.time()
return list(rows), False, db_ms