거래 빠르게 안티에서 병신만든거 커서로

feat: Implement backtest source management and enhance candle data handling

Changes:
- Introduced a new function `_apply_backtest_source_env_from_request` to manage the environment variables for candle, tick, and order book sources based on incoming requests.
- Added a teardown function `_teardown_backtest_source_env` to ensure that environment variables do not persist between requests, enhancing the stability of the backtesting environment.
- Refactored existing code to utilize the new source management functions, improving code readability and maintainability.
- Added new utility functions in `bt_candle_source.py` for fetching and managing candle data, ensuring consistency with live trading data sources.

Impact:
- These changes improve the flexibility and reliability of the backtesting framework, allowing for better management of data sources and reducing the risk of cross-request contamination.
This commit is contained in:
Your Name
2026-08-13 16:03:40 +09:00
parent c6bd62a25f
commit 2c7ad867f4
53 changed files with 15251 additions and 637 deletions

View File

@@ -28,14 +28,29 @@ from ..utils.env import get_env_bool, get_env_from_db, get_env_int
from ..utils.logger import get_logger
import logging
def _live_feed_providers() -> Tuple[str, str]:
"""시세(T)·호가(O) provider — RAM TTL 캐시(get_env_from_db). 로그 접두용.
TradeDB.get_merged_env_snapshot() 직접 호출 금지(핫패스에서 config 전체 재조회).
"""
tick_p = str(get_env_from_db("LIVE_TICK_PROVIDER", "kiwoom") or "kiwoom").strip().lower()
ob_p = str(get_env_from_db("LIVE_OB_PROVIDER", "kiwoom") or "kiwoom").strip().lower()
return tick_p, ob_p
class FeedPrefixLoggerAdapter(logging.LoggerAdapter):
"""탈락/스캔 로그(🔍 [) 앞에 T:시세|O:호가 provider 접두어를 붙인다.
예: 🔍 [탈락-RSI] → 🔍 [T:kiwoom|O:kiwoom|탈락-RSI]
(피드 출처 디버깅용 · 매매 수치 아님)
"""
def process(self, msg, kwargs):
if isinstance(msg, str) and "🔍 [" in msg and "시세:" not in msg and "LIVE_TICK_PROVIDER" in self.extra:
try:
db_obj = self.extra.get("db")
if db_obj:
tick_p = str(db_obj.get_merged_env_snapshot().get("LIVE_TICK_PROVIDER") or "kiwoom").strip().lower()
ob_p = str(db_obj.get_merged_env_snapshot().get("LIVE_OB_PROVIDER") or "kiwoom").strip().lower()
# 이미 T:|O: 접두가 있으면 중복 삽입 금지 (매수체크 로그 등)
if "🔍 [T:" not in msg[:24]:
tick_p, ob_p = _live_feed_providers()
if tick_p or ob_p:
msg = msg.replace("🔍 [", f"🔍 [T:{tick_p}|O:{ob_p}|", 1)
except Exception:
@@ -129,10 +144,12 @@ class BaseStrategy(ABC, threading.Thread):
self.universe_source = default
self._running = False
# 보유 종목 — 매 루프 DB active_trades 와 동기화 (진실의 원천 = DB)
# 보유 종목 — 재시작·이벤트·안전망(기본 60초) DB sync. 루프마다 SELECT 금지.
self.holdings: Dict[str, dict] = {}
# 장중 고점·세션저점·전략별 부가키 — DB sync 로 덮어쓰지 않음 (래칫/어깨 퇴행 방지)
self._runtime: Dict[str, dict] = {}
self._holdings_db_sync_ts: float = 0.0
self._prof_sync_skipped: int = 0
# 최근 매도 쿨다운 (종목별 마지막 매도 타임스탬프)
self.recently_sold: Dict[str, float] = {}
# 후보 ENTER 시각 (datetime) — 중분 편입 시 해당 봉 시가 매수 보류용
@@ -147,6 +164,21 @@ class BaseStrategy(ABC, threading.Thread):
self._sell_lock = threading.Lock()
self._tick_sell_last_ts: Dict[str, float] = {}
self._tick_sell_listener_on = False
# 루프 숙제별 ms 계측 (LOOP_PROFILE_ENABLED)
self._loop_prof_i = 0
self._tick_sell_lock_miss = 0
self._loop_prof_scan: Optional[Dict[str, Any]] = None
# 당일 trade_history — 루프당 1회 DB, 종목 check_buy 는 RAM 필터만
self._today_trades_cache_day: str = ""
self._today_trades_cache: List[Dict] = []
# 세밀 계측 카운터 (루프마다 리셋)
self._prof_trd_hit = 0
self._prof_trd_miss = 0
self._prof_trd_db_ms = 0.0
self._prof_sync_db_ms = 0.0
self._prof_sync_merge_ms = 0.0
self._prof_cand_load_ms = 0.0
self._prof_cand_note_ms = 0.0
self._sync_holdings_from_db(log_restore=True)
@@ -219,6 +251,7 @@ class BaseStrategy(ABC, threading.Thread):
return
self._tick_sell_last_ts[code] = now
if not self._sell_lock.acquire(blocking=False):
self._tick_sell_lock_miss = int(getattr(self, "_tick_sell_lock_miss", 0) or 0) + 1
return
try:
if code not in self.holdings:
@@ -238,6 +271,214 @@ class BaseStrategy(ABC, threading.Thread):
finally:
self._sell_lock.release()
def _loop_profile_on(self) -> bool:
"""LOOP_PROFILE_ENABLED — 한 바퀴 숙제별 ms 계측 ON/OFF."""
try:
return bool(get_env_bool("LOOP_PROFILE_ENABLED", False))
except Exception:
return False
def _loop_profile_every_n(self) -> int:
try:
return max(1, int(get_env_int("LOOP_PROFILE_EVERY_N", 1) or 1))
except Exception:
return 1
def _loop_profile_emit(self, row: Dict[str, Any]) -> None:
"""계측 1줄 — 전략 logger + 선택 파일."""
parts = [
f"[LOOP_PROF] {self.strategy_id}",
f"total={row.get('total_ms', 0):.1f}",
f"reload={row.get('reload_ms', 0):.1f}",
f"sync_hold={row.get('sync_hold_ms', 0):.1f}",
f"sync_db={row.get('sync_db_ms', 0):.1f}",
f"sync_merge={row.get('sync_merge_ms', 0):.1f}",
f"sync_skip={row.get('sync_skip', 0)}",
f"halt={row.get('halt_ms', 0):.1f}",
f"lock_wait={row.get('lock_wait_ms', 0):.1f}",
f"lock_hold={row.get('lock_hold_ms', 0):.1f}",
f"sell_chk={row.get('sell_chk_ms', 0):.1f}",
f"prefetch={row.get('prefetch_ms', 0):.1f}",
f"submit_sell={row.get('submit_sell_ms', 0):.1f}",
f"cand={row.get('cand_ms', 0):.1f}",
f"cand_load={row.get('cand_load_ms', 0):.1f}",
f"cand_mgr={row.get('cand_mgr_ms', 0):.1f}",
f"cand_post={row.get('cand_post_ms', 0):.1f}",
f"cand_slot={row.get('cand_slot_ms', 0):.1f}",
f"cand_note={row.get('cand_note_ms', 0):.1f}",
f"cand_src={row.get('cand_src', '-')}",
f"cand_n={row.get('cand_n', 0)}",
f"ws_sync={row.get('ws_sync_ms', 0):.1f}",
f"pending={row.get('pending_ms', 0):.1f}",
f"scan={row.get('scan_ms', 0):.1f}",
f"scan_other={row.get('scan_other_ms', 0):.1f}",
f"scan_name={row.get('scan_name_ms', 0):.1f}",
f"scan_pre={row.get('scan_pre_ms', 0):.1f}",
f"pre_filt={row.get('pre_filt_ms', 0):.1f}",
f"pre_guard={row.get('pre_guard_ms', 0):.1f}",
f"g_env={row.get('guard_env_ms', 0):.1f}",
f"g_daily={row.get('guard_daily_ms', 0):.1f}",
f"g_trdb={row.get('guard_trades_db_ms', 0):.1f}",
f"g_tr_n={row.get('guard_trades_n', 0)}",
f"g_budg={row.get('guard_budget_ms', 0):.1f}",
f"pre_cd={row.get('pre_cd_ms', 0):.1f}",
f"pre_gap={row.get('pre_gap_ms', 0):.1f}",
f"sleep_rej={row.get('sleep_rej_ms', 0):.1f}",
f"sleep_ok={row.get('sleep_ok_ms', 0):.1f}",
f"sleep_fail={row.get('sleep_fail_ms', 0):.1f}",
f"buy_n={row.get('buy_n', 0)}",
f"buy_sum={row.get('buy_sum_ms', 0):.1f}",
f"buy_max={row.get('buy_max_ms', 0):.1f}",
f"trd_hit={row.get('trd_hit', 0)}",
f"trd_miss={row.get('trd_miss', 0)}",
f"trd_db={row.get('trd_db_ms', 0):.1f}",
f"gap_skip={row.get('gap_skip', 0)}",
f"slot_et={row.get('slot_et_ms', 0):.1f}",
f"slot_get={row.get('slot_get_ms', 0):.1f}",
f"slot_ok={row.get('slot_ok', 0)}",
f"slot_miss={row.get('slot_miss', 0)}",
f"slot_hit={row.get('slot_cache_hit', 0)}",
f"slot_missc={row.get('slot_cache_miss', 0)}",
f"g_hit={row.get('guard_trades_hit', 0)}",
f"overlay={row.get('overlay_ms', 0):.1f}",
f"sleep={row.get('sleep_ms', 0):.1f}",
f"tick_lock_miss={row.get('tick_lock_miss', 0)}",
f"sum_parts={row.get('sum_parts_ms', 0):.1f}",
]
if row.get("buy_max_code"):
parts.append(f"buy_max_code={row.get('buy_max_code')}")
line = " ".join(parts)
try:
self.logger.info("%s", line)
except Exception:
pass
path = str(get_env_from_db("LOOP_PROFILE_LOG_PATH", "logs/loop_profile.log") or "").strip()
if not path:
return
try:
import os
if not os.path.isabs(path):
root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
path = os.path.join(root, path)
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
with open(path, "a", encoding="utf-8") as f:
f.write(dt.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + " " + line + "\n")
except Exception as ex:
try:
self.logger.debug("LOOP_PROF 파일 기록 실패: %s", ex)
except Exception:
pass
def _cb_prof_start(self, code: str) -> Optional[Dict[str, Any]]:
"""check_buy 내부 단계 계측 시작 — LOOP_PROFILE_ENABLED 일 때만."""
if not self._loop_profile_on():
return None
now = time.perf_counter()
return {
"code": str(code or ""),
"t0": now,
"_last": now,
"stages": {},
}
def _cb_prof_mark(self, ctx: Optional[Dict[str, Any]], stage: str) -> None:
if not ctx:
return
now = time.perf_counter()
last = float(ctx.get("_last") or ctx.get("t0") or now)
st = ctx.setdefault("stages", {})
st[stage] = float(st.get(stage, 0) or 0) + (now - last) * 1000.0
ctx["_last"] = now
def _cb_prof_finish(self, ctx: Optional[Dict[str, Any]], *, note: str = "") -> None:
"""check_buy 끝 — 느린 건 CHECK_BUY_PROF 로그 + 최근 결과 보관."""
if not ctx:
return
now = time.perf_counter()
total = (now - float(ctx.get("t0") or now)) * 1000.0
stages = ctx.setdefault("stages", {})
# 마지막 mark ~ finish (탈락 logger / 어댑터 등)
try:
last = float(ctx.get("_last") or ctx.get("t0") or now)
after_ms = (now - last) * 1000.0
if after_ms >= 0.05:
stages["after"] = float(stages.get("after", 0) or 0) + after_ms
except Exception:
pass
ctx["total_ms"] = total
self._last_cb_prof = ctx
try:
min_ms = float(get_env_int("CHECK_BUY_PROF_MIN_MS", 500) or 500)
except Exception:
min_ms = 500.0
if total < min_ms:
return
# 큰 단계 순
ordered = sorted(
((k, float(v or 0)) for k, v in stages.items()),
key=lambda x: -x[1],
)
parts = [
f"[CHECK_BUY_PROF] {self.strategy_id}",
f"code={ctx.get('code')}",
f"total={total:.1f}",
]
if note:
parts.append(f"note={note}")
for k, v in ordered[:12]:
parts.append(f"{k}={v:.1f}")
line = " ".join(parts)
try:
self.logger.info("%s", line)
except Exception:
pass
path = str(get_env_from_db("LOOP_PROFILE_LOG_PATH", "logs/loop_profile.log") or "").strip()
if not path:
return
try:
import os
if not os.path.isabs(path):
root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
path = os.path.join(root, path)
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
with open(path, "a", encoding="utf-8") as f:
f.write(dt.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + " " + line + "\n")
except Exception:
pass
def _run_sell_section_locked(self) -> Dict[str, float]:
"""매도 구간 — _sell_lock blocking. wait/hold/세부 ms 반환."""
out = {
"lock_wait_ms": 0.0,
"lock_hold_ms": 0.0,
"sell_chk_ms": 0.0,
"prefetch_ms": 0.0,
"submit_sell_ms": 0.0,
}
t_wait0 = time.perf_counter()
self._sell_lock.acquire(blocking=True)
out["lock_wait_ms"] = (time.perf_counter() - t_wait0) * 1000.0
t_hold0 = time.perf_counter()
try:
t0 = time.perf_counter()
sell_signals = self.check_sell_signals() or []
out["sell_chk_ms"] = (time.perf_counter() - t0) * 1000.0
if sell_signals and get_env_bool("REAL_BALANCE_VERIFY_BEFORE_SELL", True):
t1 = time.perf_counter()
try:
self.order_mgr.prefetch_broker_holdings()
except Exception:
pass
out["prefetch_ms"] = (time.perf_counter() - t1) * 1000.0
t2 = time.perf_counter()
for sig in sell_signals:
self._submit_sell(sig)
out["submit_sell_ms"] = (time.perf_counter() - t2) * 1000.0
finally:
out["lock_hold_ms"] = (time.perf_counter() - t_hold0) * 1000.0
self._sell_lock.release()
return out
# ------------------------------------------------------------------
# 스캔 루프 sleep (env 핫리로드 — 재시작 없이 반영)
# ------------------------------------------------------------------
@@ -300,6 +541,16 @@ class BaseStrategy(ABC, threading.Thread):
last_closed_log = 0.0
while self._running:
try:
prof = self._loop_profile_on()
self._loop_prof_i = int(getattr(self, "_loop_prof_i", 0) or 0) + 1
do_prof = bool(
prof
and (self._loop_prof_i % self._loop_profile_every_n() == 0)
)
row: Dict[str, Any] = {}
t_loop0 = time.perf_counter() if do_prof else 0.0
miss0 = int(getattr(self, "_tick_sell_lock_miss", 0) or 0)
now = dt.now()
today = now.strftime("%Y-%m-%d")
@@ -328,57 +579,102 @@ class BaseStrategy(ABC, threading.Thread):
continue
# 설정 리로드 (DB env_config 실시간 반영)
t0 = time.perf_counter() if do_prof else 0.0
# 세밀 계측 카운터 리셋 (당일거래 공유캐시는 루프마다 비우지 않음)
self._prof_trd_hit = 0
self._prof_trd_miss = 0
self._prof_trd_db_ms = 0.0
self._prof_sync_db_ms = 0.0
self._prof_sync_merge_ms = 0.0
self._prof_sync_skipped = 0
self._prof_cand_load_ms = 0.0
self._prof_cand_note_ms = 0.0
self._prof_cand_mgr_ms = 0.0
self._prof_cand_post_ms = 0.0
self._prof_cand_slot_ms = 0.0
self._prof_cand_src = "-"
self._prof_cand_n = 0
self._prof_slot_et_ms = 0.0
self._prof_slot_get_ms = 0.0
self._prof_slot_ok = 0
self._prof_slot_miss = 0
self._prof_slot_cache_hit = 0
self._prof_slot_cache_miss = 0
self._prof_guard_acc = {}
self.reload_config()
if do_prof:
row["reload_ms"] = (time.perf_counter() - t0) * 1000.0
# 보유 목록 = DB 진실 + _runtime 오버레이 (poll 체결·재시작 정합)
# 보유 = 이벤트 RAM + 안전망 DB sync(기본 60초). 매수/매도 체결은 RAM 즉시 갱신.
t0 = time.perf_counter() if do_prof else 0.0
self._sync_holdings_from_db()
if do_prof:
row["sync_hold_ms"] = (time.perf_counter() - t0) * 1000.0
row["sync_db_ms"] = float(self._prof_sync_db_ms)
row["sync_merge_ms"] = float(self._prof_sync_merge_ms)
row["sync_skip"] = int(getattr(self, "_prof_sync_skipped", 0) or 0)
# 전략 ON/OFF 핫게이트 — WS 구독 해제 없음. 보유 청산만 유지.
if not self._strategy_switch_enabled():
if self.holdings:
with self._sell_lock:
sell_signals = self.check_sell_signals()
if sell_signals and get_env_bool(
"REAL_BALANCE_VERIFY_BEFORE_SELL", True
):
self.order_mgr.prefetch_broker_holdings()
for sig in sell_signals:
self._submit_sell(sig)
self._run_sell_section_locked()
time.sleep(self._scan_sleep("loop"))
continue
# 일일익절 B안 — 매수루프 비어도 hit 후 보유 리스크 정리
t0 = time.perf_counter() if do_prof else 0.0
guard = getattr(self, "daily_profit_halt", None)
if guard is not None:
try:
guard.maybe_trim_open_risk(self.strategy_id)
except Exception as ex:
self.logger.debug("일일익절 리스크버짓 예외: %s", ex)
if do_prof:
row["halt_ms"] = (time.perf_counter() - t0) * 1000.0
# ── [1] 매도 먼저 ────────────────────────────────
with self._sell_lock:
sell_signals = self.check_sell_signals()
if sell_signals and get_env_bool("REAL_BALANCE_VERIFY_BEFORE_SELL", True):
self.order_mgr.prefetch_broker_holdings()
for sig in sell_signals:
self._submit_sell(sig)
sell_timings = self._run_sell_section_locked()
if do_prof:
row.update(sell_timings)
# ── [2] 후보 구독 동기화 (공유 WS) ────────────────
t0 = time.perf_counter() if do_prof else 0.0
t_c0 = time.perf_counter()
candidates = self._load_candidates()
self._prof_cand_load_ms = (time.perf_counter() - t_c0) * 1000.0
# 중분 편입 시가 애매 가드용 — 후보 ENTER 시각(초) 기록
t_n0 = time.perf_counter()
self._note_candidate_enters(candidates)
self._prof_cand_note_ms = (time.perf_counter() - t_n0) * 1000.0
if do_prof:
row["cand_ms"] = (time.perf_counter() - t0) * 1000.0
row["cand_load_ms"] = float(self._prof_cand_load_ms)
row["cand_mgr_ms"] = float(getattr(self, "_prof_cand_mgr_ms", 0) or 0)
row["cand_post_ms"] = float(getattr(self, "_prof_cand_post_ms", 0) or 0)
row["cand_slot_ms"] = float(getattr(self, "_prof_cand_slot_ms", 0) or 0)
row["cand_note_ms"] = float(self._prof_cand_note_ms)
row["cand_src"] = str(getattr(self, "_prof_cand_src", "-") or "-")
row["cand_n"] = int(getattr(self, "_prof_cand_n", 0) or 0)
cand_codes = [c.get("code") for c in candidates if c.get("code")]
hold_codes = list(self.holdings.keys())
# KIS 최소 구독 모드: 후보=키움 WS, KIS=영구+보유 (WSManager.sync_targets_split)
# US_MOMENTUM 등은 _sync_ws_for_loop 오버라이드로 해외 WS 만 사용
t0 = time.perf_counter() if do_prof else 0.0
self._sync_ws_for_loop(cand_codes, hold_codes)
if do_prof:
row["ws_sync_ms"] = (time.perf_counter() - t0) * 1000.0
# ── [2b] 미체결 지정가 만료 취소 ───────────────────
t0 = time.perf_counter() if do_prof else 0.0
self.manage_pending_orders()
if do_prof:
row["pending_ms"] = (time.perf_counter() - t0) * 1000.0
# ── [3] 매수 체크 (매수 종료 TIME_END 와 매도 세션 분리) ──
max_stocks = self._max_stocks()
active_cnt = len(self.holdings)
self._loop_prof_scan = None
t0 = time.perf_counter() if do_prof else 0.0
if candidates and active_cnt < max_stocks and self.check_buy_allowed():
self._scan_and_buy(candidates, max_stocks, active_cnt)
elif candidates and active_cnt >= max_stocks:
@@ -392,11 +688,68 @@ class BaseStrategy(ABC, threading.Thread):
active_cnt, max_stocks,
",".join(list(self.holdings.keys())[:12]),
)
if do_prof:
row["scan_ms"] = (time.perf_counter() - t0) * 1000.0
sc = getattr(self, "_loop_prof_scan", None) or {}
row["buy_n"] = int(sc.get("buy_n", 0) or 0)
row["buy_sum_ms"] = float(sc.get("buy_sum_ms", 0) or 0)
row["buy_max_ms"] = float(sc.get("buy_max_ms", 0) or 0)
row["buy_max_code"] = sc.get("buy_max_code") or ""
row["gap_skip"] = int(sc.get("gap_skip", 0) or 0)
# scan 중 check_buy 밖(이름해석·필터·sleep 등)
row["scan_other_ms"] = max(
0.0,
float(row["scan_ms"]) - float(row["buy_sum_ms"]),
)
row["scan_name_ms"] = float(sc.get("scan_name_ms", 0) or 0)
row["scan_pre_ms"] = float(sc.get("scan_pre_ms", 0) or 0)
row["pre_filt_ms"] = float(sc.get("pre_filt_ms", 0) or 0)
row["pre_guard_ms"] = float(sc.get("pre_guard_ms", 0) or 0)
row["pre_cd_ms"] = float(sc.get("pre_cd_ms", 0) or 0)
row["pre_gap_ms"] = float(sc.get("pre_gap_ms", 0) or 0)
row["sleep_rej_ms"] = float(sc.get("sleep_rej_ms", 0) or 0)
row["sleep_ok_ms"] = float(sc.get("sleep_ok_ms", 0) or 0)
row["sleep_fail_ms"] = float(sc.get("sleep_fail_ms", 0) or 0)
row["guard_env_ms"] = float(sc.get("guard_env_ms", 0) or 0)
row["guard_daily_ms"] = float(sc.get("guard_daily_ms", 0) or 0)
row["guard_trades_db_ms"] = float(sc.get("guard_trades_db_ms", 0) or 0)
row["guard_trades_n"] = int(sc.get("guard_trades_n", 0) or 0)
row["guard_budget_ms"] = float(sc.get("guard_budget_ms", 0) or 0)
row["slot_et_ms"] = float(getattr(self, "_prof_slot_et_ms", 0) or 0)
row["slot_get_ms"] = float(getattr(self, "_prof_slot_get_ms", 0) or 0)
row["slot_ok"] = int(getattr(self, "_prof_slot_ok", 0) or 0)
row["slot_miss"] = int(getattr(self, "_prof_slot_miss", 0) or 0)
row["slot_cache_hit"] = int(getattr(self, "_prof_slot_cache_hit", 0) or 0)
row["slot_cache_miss"] = int(getattr(self, "_prof_slot_cache_miss", 0) or 0)
row["guard_trades_hit"] = int(sc.get("guard_trades_hit", 0) or 0)
row["trd_hit"] = int(self._prof_trd_hit)
row["trd_miss"] = int(self._prof_trd_miss)
row["trd_db_ms"] = float(self._prof_trd_db_ms)
# 고점·세션저점 등 런타임 오버레이 저장 (다음 루프 DB sync 시 max merge)
t0 = time.perf_counter() if do_prof else 0.0
self._capture_runtime_overlay()
if do_prof:
row["overlay_ms"] = (time.perf_counter() - t0) * 1000.0
t0 = time.perf_counter() if do_prof else 0.0
time.sleep(self._scan_sleep("loop"))
if do_prof:
row["sleep_ms"] = (time.perf_counter() - t0) * 1000.0
row["total_ms"] = (time.perf_counter() - t_loop0) * 1000.0
row["tick_lock_miss"] = int(
getattr(self, "_tick_sell_lock_miss", 0) or 0
) - miss0
# 숙제 합( sleep 제외 ) — total 과 비교해 미계측 구간 파악
part_keys = (
"reload_ms", "sync_hold_ms", "halt_ms",
"lock_wait_ms", "lock_hold_ms",
"cand_ms", "ws_sync_ms", "pending_ms",
"scan_ms", "overlay_ms",
)
# lock_hold 안에 sell_chk/prefetch/submit 포함 → 합산 시 hold만
row["sum_parts_ms"] = sum(float(row.get(k, 0) or 0) for k in part_keys)
self._loop_profile_emit(row)
except KeyboardInterrupt:
self._running = False
@@ -480,8 +833,15 @@ class BaseStrategy(ABC, threading.Thread):
return None
ms = max_stocks if max_stocks is not None else self._max_stocks()
slot = float(getattr(self, "slot_money", 0) or get_env_int("SLOT_MONEY_DEFAULT", 3_000_000))
prof = None
if self._loop_profile_on():
prof = getattr(self, "_prof_guard_acc", None)
if prof is None:
prof = {}
self._prof_guard_acc = prof
return live_portfolio_entry_reject(
self.db, self.holdings, self.strategy_id, code, slot, ms,
prof=prof,
)
def _resolve_live_buy_qty(
@@ -553,59 +913,176 @@ class BaseStrategy(ABC, threading.Thread):
for c in (candidates or [])
]
_codes = [c for c in _codes if c]
_tick_p, _ob_p = _live_feed_providers()
if _codes and str(getattr(self, "strategy_id", "")).upper().startswith("US_"):
self.logger.info(
"🔍 [매수체크/T:%s|O:%s] 후보 %d (보유 %d/%d) codes=%s",
self.db.get_merged_env_snapshot().get("LIVE_TICK_PROVIDER", "kiwoom"),
self.db.get_merged_env_snapshot().get("LIVE_OB_PROVIDER", "kiwoom"),
_tick_p, _ob_p,
len(candidates), active_cnt, max_stocks,
",".join(_codes[:12]),
)
else:
self.logger.info(
"🔍 [매수체크/T:%s|O:%s] 후보 %d (보유 %d/%d)",
self.db.get_merged_env_snapshot().get("LIVE_TICK_PROVIDER", "kiwoom"),
self.db.get_merged_env_snapshot().get("LIVE_OB_PROVIDER", "kiwoom"),
_tick_p, _ob_p,
len(candidates), active_cnt, max_stocks,
)
prof_scan = self._loop_profile_on()
buy_n = 0
buy_sum = 0.0
buy_max = 0.0
buy_max_code = ""
gap_skip = 0
scan_name_ms = 0.0
scan_pre_ms = 0.0
pre_filt_ms = 0.0
pre_guard_ms = 0.0
pre_cd_ms = 0.0
pre_gap_ms = 0.0
sleep_rej_ms = 0.0
sleep_ok_ms = 0.0
sleep_fail_ms = 0.0
if prof_scan:
self._prof_guard_acc = {}
def _snap_scan():
g = getattr(self, "_prof_guard_acc", None) or {}
return {
"buy_n": buy_n,
"buy_sum_ms": buy_sum,
"buy_max_ms": buy_max,
"buy_max_code": buy_max_code,
"gap_skip": gap_skip,
"scan_name_ms": scan_name_ms,
"scan_pre_ms": scan_pre_ms,
"pre_filt_ms": pre_filt_ms,
"pre_guard_ms": pre_guard_ms,
"pre_cd_ms": pre_cd_ms,
"pre_gap_ms": pre_gap_ms,
"sleep_rej_ms": sleep_rej_ms,
"sleep_ok_ms": sleep_ok_ms,
"sleep_fail_ms": sleep_fail_ms,
"guard_env_ms": float(g.get("guard_env_ms", 0) or 0),
"guard_daily_ms": float(g.get("guard_daily_ms", 0) or 0),
"guard_trades_db_ms": float(g.get("guard_trades_db_ms", 0) or 0),
"guard_trades_n": int(g.get("guard_trades_n", 0) or 0),
"guard_trades_hit": int(g.get("guard_trades_hit", 0) or 0),
"guard_budget_ms": float(g.get("guard_budget_ms", 0) or 0),
}
for c in candidates:
if not self._running:
if prof_scan:
self._loop_prof_scan = _snap_scan()
return
t_pre0 = time.perf_counter() if prof_scan else 0.0
code = c.get("code") or c.get("stk_cd", "")
name = c.get("name") or c.get("stk_nm", code)
if name == code or not str(name or "").strip():
try:
from ..utils.stock_name import resolve_stock_display_name
name = resolve_stock_display_name(self.db, code, name)
except Exception:
name = code
# 매수체크 핫패스: DB 이름조회 금지(느림·불필요).
# 한글명은 주문/알림 시 OrderManager._resolve_order_display_name 만.
name = c.get("name") or c.get("stk_nm") or code
if not str(name or "").strip():
name = code
if prof_scan:
scan_name_ms += (time.perf_counter() - t_pre0) * 1000.0
t_pre0 = time.perf_counter()
if not code or code in self.holdings:
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_filt_ms += dt
continue
if code in self.untradable_skip:
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_filt_ms += dt
continue
if not self._candidate_filter(c):
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_filt_ms += dt
continue
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_filt_ms += dt
t_pre0 = time.perf_counter()
guard = self._live_portfolio_entry_guard(code, max_stocks)
if guard:
self.logger.info("🔍 [%s] %s(%s)", guard, name, code)
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_guard_ms += dt
continue
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_guard_ms += dt
t_pre0 = time.perf_counter()
# 재진입 쿨다운
cooldown_sec = self._reentry_cooldown_sec()
elapsed = time.time() - self.recently_sold.get(code, 0)
if elapsed < cooldown_sec:
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_cd_ms += dt
continue
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_cd_ms += dt
t_pre0 = time.perf_counter()
# 갭보정 미완료 → 불완전한 봉으로 매수 판단 금지
# 갭보정 완료 후 자동으로 다음 루프에서 check_buy 진입
if hasattr(self.ws, "is_gap_ready") and not self.ws.is_gap_ready(code):
gap_skip += 1
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_gap_ms += dt
continue
if prof_scan:
dt = (time.perf_counter() - t_pre0) * 1000.0
scan_pre_ms += dt
pre_gap_ms += dt
t_buy0 = time.perf_counter() if prof_scan else 0.0
signal = self.check_buy(code, name)
if prof_scan:
dt_ms = (time.perf_counter() - t_buy0) * 1000.0
buy_n += 1
buy_sum += dt_ms
if dt_ms >= buy_max:
buy_max = dt_ms
buy_max_code = str(code)
if not signal:
if prof_scan:
t_sl = time.perf_counter()
time.sleep(self._scan_sleep("reject"))
if prof_scan:
sleep_rej_ms += (time.perf_counter() - t_sl) * 1000.0
continue
result = self._submit_buy(signal)
if result and result.success:
if prof_scan:
t_sl = time.perf_counter()
time.sleep(self._scan_sleep("buy_ok"))
if prof_scan:
sleep_ok_ms += (time.perf_counter() - t_sl) * 1000.0
self._loop_prof_scan = _snap_scan()
return # 1루프당 1매수 (포지션 과집중 방지)
if prof_scan:
t_sl = time.perf_counter()
time.sleep(self._scan_sleep("buy_fail"))
if prof_scan:
sleep_fail_ms += (time.perf_counter() - t_sl) * 1000.0
if prof_scan:
self._loop_prof_scan = _snap_scan()
def _note_candidate_enters(self, candidates: List[Dict]) -> None:
"""후보 집합 변화 → ENTER/EXIT 시각 갱신 (중분 시가 애매 가드)."""
@@ -702,6 +1179,11 @@ class BaseStrategy(ABC, threading.Thread):
"size_class": req.size_class or "",
}
self._capture_runtime_overlay()
try:
from ..utils.today_trades_cache import invalidate_today_trades_cache
invalidate_today_trades_cache()
except Exception:
pass
elif result.success and signal.get("use_limit_buy"):
self.on_limit_buy_submitted(signal, result)
else:
@@ -860,14 +1342,38 @@ class BaseStrategy(ABC, threading.Thread):
)
def _load_holdings_from_db(self, *, log_restore: bool = False) -> None:
"""DB → holdings 동기화 (지정가 체결 등 이벤트 시 호출)."""
self._sync_holdings_from_db(log_restore=log_restore)
"""DB → holdings 강제 동기화 (지정가 체결·복원 등 이벤트 시)."""
self._sync_holdings_from_db(log_restore=log_restore, force=True)
def _drop_local_position(self, code: str) -> None:
"""매도·유령정리 후 메모리 보유·런타임 오버레이 제거."""
self.holdings.pop(code, None)
self._runtime.pop(code, None)
def _should_skip_holdings_db_sync(self, *, force: bool, log_restore: bool) -> bool:
"""하이브리드 안전망: 간격 미경과면 DB get_active_trades 생략.
HOLDINGS_DB_SYNC_INTERVAL_SEC=0 → 매 루프 sync(레거시).
force/log_restore → 항상 수행.
"""
if force or log_restore:
return False
try:
interval = max(0, int(get_env_int("HOLDINGS_DB_SYNC_INTERVAL_SEC", 60) or 0))
except Exception:
interval = 60
if interval <= 0:
return False
last = float(getattr(self, "_holdings_db_sync_ts", 0.0) or 0.0)
if last <= 0.0:
return False
if (time.time() - last) < float(interval):
self._prof_sync_db_ms = 0.0
self._prof_sync_merge_ms = 0.0
self._prof_sync_skipped = 1
return True
return False
def _merge_runtime_overlay(
self, code: str, avg_bp: float, db_max: float, db_sess_low: float,
) -> Tuple[float, float]:
@@ -918,17 +1424,25 @@ class BaseStrategy(ABC, threading.Thread):
"""서브클래스 훅 — DB sync 직후 (UPDOW entry_bar_key 등)."""
return None
def _sync_holdings_from_db(self, *, log_restore: bool = False) -> None:
"""DB active_trades → holdings (진실의 원천). 장중 고점은 _runtime 과 merge.
def _sync_holdings_from_db(self, *, log_restore: bool = False, force: bool = False) -> None:
"""DB active_trades → holdings. 하이브리드: 이벤트 RAM + 주기 안전망.
- DB에 없는 종목은 holdings·_runtime 에서 제거 (양방향 정합)
- poll_pending 매수 체결·재시작 후에도 다음 루프에 자동 반영
- 재시작(log_restore) / force / 지정가 체결(_load_holdings_from_db): 즉시 DB
- 루프: HOLDINGS_DB_SYNC_INTERVAL_SEC(기본 60)마다만 DB (공유 conn 락 완화)
- 시장가 매수·매도 성공 시 RAM은 이미 갱신 (_submit_buy / _drop_local_position)
- 장중 고점은 _runtime 과 merge. DB에 없는 종목은 holdings·_runtime 제거
- ETF/ETN/스팩 등 시세 불가 종목만 보유 루프에서 제외.
우선주는 매수 차단 대상이지만, 이미 산 경우 매도 가능하도록 유지.
"""
if self._should_skip_holdings_db_sync(force=force, log_restore=log_restore):
return
self._prof_sync_skipped = 0
try:
prefix = self.strategy_id.split("_")[0] if "_" in self.strategy_id else self.strategy_id
t_db0 = time.perf_counter()
rows = self.db.get_active_trades(strategy_prefix=prefix)
self._prof_sync_db_ms = (time.perf_counter() - t_db0) * 1000.0
t_m0 = time.perf_counter()
skipped_non_stock: list[str] = []
new_holdings: Dict[str, dict] = {}
for code, t in rows.items():
@@ -972,6 +1486,8 @@ class BaseStrategy(ABC, threading.Thread):
self.holdings.clear()
self.holdings.update(new_holdings)
self._after_holdings_sync()
self._prof_sync_merge_ms = (time.perf_counter() - t_m0) * 1000.0
self._holdings_db_sync_ts = time.time()
if log_restore and self.holdings:
self.logger.info(
@@ -1046,6 +1562,11 @@ class BaseStrategy(ABC, threading.Thread):
]
candidates: List[Dict] = []
do_prof = self._loop_profile_on()
self._prof_cand_src = current_src
self._prof_cand_mgr_ms = 0.0
self._prof_cand_post_ms = 0.0
self._prof_cand_slot_ms = 0.0
for tag in order:
mgr = mgr_by_src.get(tag)
if mgr is None:
@@ -1053,6 +1574,7 @@ class BaseStrategy(ABC, threading.Thread):
if not self._is_strategy_registered(mgr):
continue
try:
t_m0 = time.perf_counter() if do_prof else 0.0
universe = mgr.get_universe_for(self.strategy_id)
# 매니저에 등록은 돼 있으나 아직 비어있음 → [] 이 정답 (폴백 금지).
# (설정된 유니버스가 비어있을 수 있는 상태: 장 마감 후 등)
@@ -1060,20 +1582,36 @@ class BaseStrategy(ABC, threading.Thread):
candidates = mgr.get_candidates_for(self.strategy_id) or []
else:
candidates = []
return self._post_filter_candidates(candidates)
if do_prof:
self._prof_cand_mgr_ms = (time.perf_counter() - t_m0) * 1000.0
self._prof_cand_src = tag
out = self._post_filter_candidates(candidates)
if do_prof:
self._prof_cand_n = len(out)
return out
except Exception as e:
self.logger.debug("%s 유니버스 로드 실패: %s", tag, e)
# 매니저 모두 본 전략 설정 없음 → 레거시 DB 경로
# (ls_condition 은 DB 폴백도 하지 않음 — 빈 후보)
if self.universe_source == "ls_condition":
return self._post_filter_candidates([])
out = self._post_filter_candidates([])
if do_prof:
self._prof_cand_n = len(out)
return out
try:
t_m0 = time.perf_counter() if do_prof else 0.0
candidates = self.db.get_target_candidates() or []
if do_prof:
self._prof_cand_mgr_ms = (time.perf_counter() - t_m0) * 1000.0
self._prof_cand_src = "db_legacy"
except Exception as e:
self.logger.debug("target_candidates 조회 실패: %s", e)
candidates = []
return self._post_filter_candidates(candidates)
out = self._post_filter_candidates(candidates)
if do_prof:
self._prof_cand_n = len(out)
return out
def _post_filter_candidates(self, candidates: List[Dict]) -> List[Dict]:
"""매니저 결과 → ETN/비본주 자동 제외 + 전략별 후보 하드캡.
@@ -1084,7 +1622,12 @@ class BaseStrategy(ABC, threading.Thread):
WS 구독 41 한도 안전 + cond/ranking 폭주 시 매수 체크 분당 회전율 보장.
(조건검색 매니저는 신규 ENTER·t1859 스냅을 앞으로 두어 캡에 신규가 안 잘리게 함)
"""
do_prof = self._loop_profile_on()
t_post0 = time.perf_counter() if do_prof else 0.0
if not candidates:
if do_prof:
self._prof_cand_post_ms = (time.perf_counter() - t_post0) * 1000.0
self._prof_cand_slot_ms = 0.0
return []
# 1) 비본주 필터
@@ -1119,19 +1662,31 @@ class BaseStrategy(ABC, threading.Thread):
live_universe_slot_align_enabled,
resolve_live_universe_history_source,
)
slot_ms = 0.0
if live_universe_slot_align_enabled(sid):
before = len(candidates)
univ_src = str(getattr(self, "universe_source", "") or "")
hist_src = resolve_live_universe_history_source(
sid, universe_source=univ_src,
)
t_slot0 = time.perf_counter() if do_prof else 0.0
slot_prof: Dict[str, float] = {} if do_prof else None # type: ignore[assignment]
candidates, dropped = filter_candidates_by_history_universe(
candidates,
self.db,
sid,
universe_source=univ_src,
history_source=hist_src,
prof=slot_prof,
)
if do_prof:
slot_ms = (time.perf_counter() - t_slot0) * 1000.0
self._prof_slot_et_ms = float((slot_prof or {}).get("slot_et_ms", 0) or 0)
self._prof_slot_get_ms = float((slot_prof or {}).get("slot_get_ms", 0) or 0)
self._prof_slot_ok = int((slot_prof or {}).get("slot_ok", 0) or 0)
self._prof_slot_miss = int((slot_prof or {}).get("slot_miss", 0) or 0)
self._prof_slot_cache_hit = int((slot_prof or {}).get("slot_cache_hit", 0) or 0)
self._prof_slot_cache_miss = int((slot_prof or {}).get("slot_cache_miss", 0) or 0)
if dropped > 0:
now_ts = time.time()
last = getattr(self, "_universe_slot_log_ts", 0.0)
@@ -1155,6 +1710,9 @@ class BaseStrategy(ABC, threading.Thread):
except Exception:
pass
if do_prof:
self._prof_cand_post_ms = (time.perf_counter() - t_post0) * 1000.0
self._prof_cand_slot_ms = slot_ms
return candidates
def _is_strategy_registered(self, mgr) -> bool:
@@ -1171,6 +1729,25 @@ class BaseStrategy(ABC, threading.Thread):
"""날짜 변경 훅 (서브클래스 오버라이드 가능)."""
pass
def _get_today_trades(self, today: Optional[str] = None) -> List[Dict]:
"""당일 trade_history — 전략 공유 RAM + TTL (종목·전략마다 SELECT 금지).
일일 횟수/손익 게이트·pre_guard 가 동일 캐시 사용.
TTL(기본 1초) 내·같은 날짜면 DB 안 감. 매수 체결 시 invalidate.
"""
from ..utils.today_trades_cache import get_today_trades_cached
day = str(today or dt.now().strftime("%Y%m%d"))
rows, from_cache, db_ms = get_today_trades_cached(self.db, day)
if from_cache:
self._prof_trd_hit += 1
else:
self._prof_trd_miss += 1
self._prof_trd_db_ms += float(db_ms or 0)
self._today_trades_cache_day = day
self._today_trades_cache = rows
return rows
def reload_config(self) -> None:
"""설정 리로드 훅 (서브클래스 오버라이드)."""
pass