hoon/kis_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 틱/호가 데이터 출처 표출 UI 추가 및 데이터 수집 개선

Browse Source 
- 백테스트 및 실거래 시 틱과 호가의 벤더 출처(ob_source, entry_source) 기록 및 추적 강화 (tail_engine.py)
- 웹 UI '체결디버그'에 [틱:kis / 호가:ls] 형태로 데이터 출처를 직관적으로 표출 (backtest.js, backtest.html)
- LS WebSocket 구독 100건 제한 하드코딩 해제 및 env_config_ext 연동 (ls_ws.py)
- 기타 백테스트 웹 및 DB 관련 최적화 적용
This commit is contained in:
Your Name
2026-09-02 20:53:16 +09:00
parent 2c37771a16
commit 253c95e2c2
11 changed files with 1141 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

659
kis_trader/web/feed_collect_stats.py
View File

@@ -8,9 +8,15 @@
"""
from __future__ import annotations
import time as _time
from datetime import datetime
from typing import Any, Dict, List, Optional, Set
# 통계 결과 메모리 캐시 (key: day8, value: (ts, result))
# 웹 탭 재조회 시 매번 100초대 쉷 나는 일일 직접 스캔을 피하기 위해 TTL 캠시 적용.
_STATS_CACHE: Dict[str, tuple] = {} # {day8: (expire_ts, result)}
_STATS_CACHE_TTL = float(300) # 5분 (ENV으로 오버라이드 가능)
def _ymd8(day: str) -> str:
s = (day or "").strip().replace("-", "")[:8]
@@ -23,6 +29,158 @@ def _like(day8: str) -> str:
return f"{day8}%"
def _day_bounds(day8: str) -> tuple:
"""일자 범위 tick_time/snap_time (VARCHAR14 숫자문자열 — LIKE 대신 range)."""
return f"{day8}000000", f"{day8}235959"
class _FeedStatsDaySlice:
"""하루치 ws_ticks/ws_orderbook/ls_ws_ticks/ls_ws_orderbook TEMP.
lag·pick 집계는 이 TEMP만 스캔 → LS 풀스캔 제거.
ls_ws_ticks TEMP는 생성 시 lag_sec(TIMESTAMPDIFF)까지 미리 계산해 저장.
"""
TICK_TMP = "tmp_fs_ticks"
OB_TMP = "tmp_fs_ob"
LS_TICK_TMP = "tmp_fs_ls_ticks"
LS_OB_TMP = "tmp_fs_ls_ob"
def __init__(self, db, day8: str) -> None:
self.db = db
self.day8 = day8
self.t0, self.t1 = _day_bounds(day8)
self.like = _like(day8)
self.ready = False
self.ls_ready = False # LS TEMP 별도 플래그 (LS WS 꺼진 날도 KIS/키움 TEMP는 동작)
self.tick_n = 0
self.ob_n = 0
self.ls_tick_n = 0
self.ls_ob_n = 0
# ls_ws_ticks 당일 ts 범위 (DATETIME 형)
from datetime import datetime as _dt
self._ls_d0 = _dt.strptime(day8, "%Y%m%d")
self._ls_d1 = self._ls_d0.replace(hour=23, minute=59, second=59)
def ensure(self) -> bool:
"""KIS/키움 TEMP + LS TEMP 모두 생성. 실패해도 KIS/키움 TEMP는 유지."""
self._ensure_kis_kiwoom()
self._ensure_ls()
return self.ready
def _ensure_kis_kiwoom(self) -> bool:
if self.ready:
return True
conn = self.db.conn
try:
conn.execute(f"DROP TEMPORARY TABLE IF EXISTS {self.TICK_TMP}")
conn.execute(f"DROP TEMPORARY TABLE IF EXISTS {self.OB_TMP}")
conn.execute(
f"CREATE TEMPORARY TABLE {self.TICK_TMP} ("
"id BIGINT, code VARCHAR(32), tick_time VARCHAR(14), "
"tick_time_raw VARCHAR(64), source VARCHAR(16), recv_ts VARCHAR(30), "
"KEY idx_src (source), KEY idx_recv (recv_ts)"
") AS "
"SELECT id, code, tick_time, tick_time_raw, source, recv_ts "
"FROM ws_ticks WHERE tick_time >= %s AND tick_time <= %s",
(self.t0, self.t1),
)
conn.execute(
f"CREATE TEMPORARY TABLE {self.OB_TMP} ("
"id BIGINT, code VARCHAR(32), snap_time VARCHAR(14), "
"source VARCHAR(16), recv_ts VARCHAR(30), "
"reject_code VARCHAR(64), strategy VARCHAR(64), "
"KEY idx_src (source)"
") AS "
"SELECT id, code, snap_time, source, recv_ts, reject_code, strategy "
"FROM ws_orderbook WHERE snap_time >= %s AND snap_time <= %s",
(self.t0, self.t1),
)
r = conn.execute(f"SELECT COUNT(*) AS n FROM {self.TICK_TMP}").fetchone()
self.tick_n = int((r.get("n") if isinstance(r, dict) else r[0]) or 0)
r = conn.execute(f"SELECT COUNT(*) AS n FROM {self.OB_TMP}").fetchone()
self.ob_n = int((r.get("n") if isinstance(r, dict) else r[0]) or 0)
self.ready = True
return True
except Exception:
self.ready = False
return False
def _ensure_ls(self) -> bool:
"""ls_ws_ticks/ls_ws_orderbook 당일치 TEMP 생성 (lag_sec 미리 계산 포함)."""
if self.ls_ready:
return True
conn = self.db.conn
d0, d1 = self._ls_d0, self._ls_d1
try:
conn.execute(f"DROP TEMPORARY TABLE IF EXISTS {self.LS_TICK_TMP}")
# ls_ws_ticks: tk, lag_sec 가 이미 계산되어 있다면 바로 사용 (폴백 지원)
conn.execute(
f"CREATE TEMPORARY TABLE {self.LS_TICK_TMP} ("
"code VARCHAR(20), tk CHAR(14), lag_sec DOUBLE, "
"KEY idx_code_tk (code, tk)"
") AS "
"SELECT code, "
" IFNULL(tk, CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), RIGHT(IFNULL(chetime,'000000'),6))) AS tk, "
" IFNULL(lag_sec, TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE("
" CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), RIGHT(IFNULL(chetime,'000000'),6)),"
" '%%Y%%m%%d%%H%%i%%s'), "
" ts)) AS lag_sec "
"FROM ls_ws_ticks WHERE ts >= %s AND ts <= %s",
(d0, d1),
)
r = conn.execute(f"SELECT COUNT(*) AS n FROM {self.LS_TICK_TMP}").fetchone()
self.ls_tick_n = int((r.get("n") if isinstance(r, dict) else r[0]) or 0)
except Exception:
self.ls_tick_n = 0
try:
conn.execute(f"DROP TEMPORARY TABLE IF EXISTS {self.LS_OB_TMP}")
# ls_ws_orderbook: tk, lag_sec 활용
conn.execute(
f"CREATE TEMPORARY TABLE {self.LS_OB_TMP} ("
"code VARCHAR(32), snap_time CHAR(14), lag_sec DOUBLE, "
"KEY idx_code_snap (code, snap_time)"
") AS "
"SELECT code, IFNULL(tk, LEFT(snap_time,14)) AS snap_time, "
" IFNULL(lag_sec, TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE(LEFT(snap_time,14), '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts)) AS lag_sec "
"FROM ls_ws_orderbook WHERE snap_time >= %s AND snap_time <= %s",
(self.t0, self.t1),
)
r = conn.execute(f"SELECT COUNT(*) AS n FROM {self.LS_OB_TMP}").fetchone()
self.ls_ob_n = int((r.get("n") if isinstance(r, dict) else r[0]) or 0)
except Exception:
self.ls_ob_n = 0
self.ls_ready = True
return True
def tick_from(self) -> str:
return self.TICK_TMP if self.ready else "ws_ticks"
def ob_from(self) -> str:
return self.OB_TMP if self.ready else "ws_orderbook"
def ls_tick_from(self) -> str:
return self.LS_TICK_TMP if self.ls_ready else None
def ls_ob_from(self) -> str:
return self.LS_OB_TMP if self.ls_ready else None
def tick_filter_sql(self) -> tuple:
if self.ready:
return "", ()
return " WHERE tick_time >= %s AND tick_time <= %s", (self.t0, self.t1)
def ob_filter_sql(self) -> tuple:
if self.ready:
return "", ()
return " WHERE snap_time >= %s AND snap_time <= %s", (self.t0, self.t1)
def _row_nc(r: Any) -> Dict[str, Any]:
if r is None:
return {"n": 0, "codes": 0}
@@ -104,11 +262,17 @@ def _pick_winner(flags: Dict[str, bool], chain: List[str]) -> Optional[str]:
def _count_tick_picks(
db, day8: str, age: float, chain: List[str],
db,
day8: str,
age: float,
chain: List[str],
day_slice: Optional[_FeedStatsDaySlice] = None,
) -> Dict[str, int]:
"""종목×초(YYYYMMDDHHMMSS)마다 usable 벤더 중 체인 1순위 간택 횟수."""
wins = {"kis": 0, "kiwoom": 0, "ls": 0}
like = _like(day8)
ds = day_slice or _FeedStatsDaySlice(db, day8)
tick_tbl = ds.tick_from()
tick_where, tick_params = ds.tick_filter_sql()
exch_dt = _sql_ws_tick_exchange_dt()
slots: Dict[tuple, Dict[str, bool]] = {}
try:
@@ -118,10 +282,10 @@ def _count_tick_picks(
"FROM ("
" SELECT code, tick_time, source, "
f" TIMESTAMPDIFF(SECOND, {exch_dt}, recv_ts) AS lag_sec "
" FROM ws_ticks WHERE tick_time LIKE %s"
f" FROM {tick_tbl}{tick_where}"
") x GROUP BY code, LEFT(tick_time, 14), source"
)
for r in db.conn.execute(sql, (age, like)).fetchall() or []:
for r in db.conn.execute(sql, (age, *tick_params)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
src = str(d.get("source") or "").strip().lower()
if src not in wins:
@@ -135,34 +299,50 @@ def _count_tick_picks(
except Exception:
pass
try:
d0 = datetime.strptime(day8, "%Y%m%d")
d1 = d0.replace(hour=23, minute=59, second=59)
ls_sql = (
"SELECT code, DATE_FORMAT(ts, '%%Y%%m%%d%%H%%i%%s') AS tk, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
"FROM ("
" SELECT code, ts, "
" TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE("
" CASE WHEN CHAR_LENGTH(REGEXP_REPLACE(IFNULL(chetime,''), '[^0-9]', ''))>=14 "
" THEN LEFT(REGEXP_REPLACE(chetime, '[^0-9]', ''), 14) "
" WHEN CHAR_LENGTH(REGEXP_REPLACE(IFNULL(chetime,''), '[^0-9]', ''))>=6 "
" THEN CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), "
" RIGHT(REGEXP_REPLACE(chetime, '[^0-9]', ''), 6)) "
" ELSE DATE_FORMAT(ts,'%%Y%%m%%d%%H%%i%%s') END, "
" '%%Y%%m%%d%%H%%i%%s'), "
" ts) AS lag_sec "
" FROM ls_ws_ticks WHERE ts>=%s AND ts<=%s"
") t GROUP BY code, DATE_FORMAT(ts, '%%Y%%m%%d%%H%%i%%s')"
)
for r in db.conn.execute(ls_sql, (age, d0, d1)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("ok") or 0) <= 0:
continue
key = (str(d.get("code") or ""), str(d.get("tk") or ""))
if not key[0] or len(key[1]) < 14:
continue
slots.setdefault(key, {})["ls"] = True
ls_tbl = ds.ls_tick_from()
if ls_tbl:
# TEMP: tk 콜럼 이미 계산(YYYYMMDDHHMMSS), lag_sec 이미 계산
ls_sql = (
"SELECT code, tk, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
f"FROM {ls_tbl} "
"GROUP BY code, tk"
)
for r in db.conn.execute(ls_sql, (age,)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("ok") or 0) <= 0:
continue
key = (str(d.get("code") or ""), str(d.get("tk") or ""))
if not key[0] or len(key[1]) < 14:
continue
slots.setdefault(key, {})["ls"] = True
else:
# TEMP 없으면 본 테이블 폴백 (REGEXP 제거한 단순 CONCAT 버전)
d0 = datetime.strptime(day8, "%Y%m%d")
d1 = d0.replace(hour=23, minute=59, second=59)
ls_sql = (
"SELECT code, "
" CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), RIGHT(IFNULL(chetime,'000000'),6)) AS tk, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
"FROM ("
" SELECT code, "
" IFNULL(tk, CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), RIGHT(IFNULL(chetime,'000000'),6))) AS tk, "
" IFNULL(lag_sec, TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE("
" CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), RIGHT(IFNULL(chetime,'000000'),6)),"
" '%%Y%%m%%d%%H%%i%%s'), "
" ts)) AS lag_sec "
" FROM ls_ws_ticks WHERE ts>=%s AND ts<=%s"
") t GROUP BY code, tk"
)
for r in db.conn.execute(ls_sql, (age, d0, d1)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("ok") or 0) <= 0:
continue
key = (str(d.get("code") or ""), str(d.get("tk") or ""))
if not key[0] or len(key[1]) < 14:
continue
slots.setdefault(key, {})["ls"] = True
except Exception:
pass
for flags in slots.values():
@@ -173,13 +353,26 @@ def _count_tick_picks(
def _count_ob_picks(
db, day8: str, age: float, chain: List[str],
db,
day8: str,
age: float,
chain: List[str],
day_slice: Optional[_FeedStatsDaySlice] = None,
) -> Dict[str, int]:
"""종목×초마다 호가 usable 벤더 체인 간택."""
wins = {"kis": 0, "kiwoom": 0, "ls": 0}
like = _like(day8)
ds = day_slice or _FeedStatsDaySlice(db, day8)
ob_tbl = ds.ob_from()
ob_where, ob_params = ds.ob_filter_sql()
t0, t1 = ds.t0, ds.t1
slots: Dict[tuple, Dict[str, bool]] = {}
try:
if ds.ready:
ob_src_where = f" FROM {ob_tbl} WHERE source<>%s"
ob_src_params: tuple = ("filter_eval",)
else:
ob_src_where = f" FROM {ob_tbl}{ob_where} AND source<>%s"
ob_src_params = (*ob_params, "filter_eval")
sql = (
"SELECT code, tk, source, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
@@ -195,11 +388,13 @@ def _count_ob_picks(
" ELSE NULL END, "
" '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts) AS lag_sec "
" FROM ws_orderbook WHERE snap_time LIKE %s AND source<>%s"
f"{ob_src_where}"
") x WHERE tk IS NOT NULL "
"GROUP BY code, tk, source"
)
for r in db.conn.execute(sql, (age, day8, day8, like, "filter_eval")).fetchall() or []:
for r in db.conn.execute(
sql, (age, day8, day8, *ob_src_params)
).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
vend = _ob_src_to_vendor(str(d.get("source") or ""))
if not vend or int(d.get("ok") or 0) <= 0:
@@ -211,25 +406,43 @@ def _count_ob_picks(
except Exception:
pass
try:
ls_sql = (
"SELECT code, LEFT(snap_time,14) AS tk, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
"FROM ("
" SELECT code, snap_time, "
" TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE(LEFT(snap_time,14), '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts) AS lag_sec "
" FROM ls_ws_orderbook WHERE snap_time LIKE %s"
") t GROUP BY code, LEFT(snap_time,14)"
)
for r in db.conn.execute(ls_sql, (age, like)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("ok") or 0) <= 0:
continue
key = (str(d.get("code") or ""), str(d.get("tk") or ""))
if not key[0] or len(key[1]) < 14:
continue
slots.setdefault(key, {})["ls"] = True
ls_ob_tbl = ds.ls_ob_from()
if ls_ob_tbl:
# TEMP 사용: lag_sec 이미 계산됨
ls_sql = (
"SELECT code, snap_time AS tk, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
f"FROM {ls_ob_tbl} "
"GROUP BY code, snap_time"
)
for r in db.conn.execute(ls_sql, (age,)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("ok") or 0) <= 0:
continue
key = (str(d.get("code") or ""), str(d.get("tk") or ""))
if not key[0] or len(key[1]) < 14:
continue
slots.setdefault(key, {})["ls"] = True
else:
ls_sql = (
"SELECT code, LEFT(snap_time,14) AS tk, "
"MAX(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END) AS ok "
"FROM ("
" SELECT code, IFNULL(tk, LEFT(snap_time,14)) AS tk, "
" IFNULL(lag_sec, TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE(LEFT(snap_time,14), '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts)) AS lag_sec "
" FROM ls_ws_orderbook WHERE snap_time >= %s AND snap_time <= %s"
") t GROUP BY code, tk"
)
for r in db.conn.execute(ls_sql, (age, t0, t1)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("ok") or 0) <= 0:
continue
key = (str(d.get("code") or ""), str(d.get("tk") or ""))
if not key[0] or len(key[1]) < 14:
continue
slots.setdefault(key, {})["ls"] = True
except Exception:
pass
for flags in slots.values():
@@ -311,7 +524,12 @@ def _sql_ws_tick_exchange_dt() -> str:
)
def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
def _feed_age_cut_stats(
db,
day8: str,
age_sec: float,
day_slice: Optional[_FeedStatsDaySlice] = None,
) -> Dict[str, Any]:
"""recv_ts vs 체결/스냅 시각 지연 → 3초(폴백나이) 합격·탈락·미상.
옵투나 ``merge_ticks_time_axis_fallback`` / 호가 lag 컷과 동일 축.
@@ -319,7 +537,12 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
empty_minutes = 그 분 틱이 전부 fail 인 (code,분) 수 (비어 재현되는 분).
"""
age = max(0.0, float(age_sec or 3.0))
like = _like(day8)
ds = day_slice or _FeedStatsDaySlice(db, day8)
tick_tbl = ds.tick_from()
ob_tbl = ds.ob_from()
tick_where, tick_params = ds.tick_filter_sql()
ob_where, ob_params = ds.ob_filter_sql()
t0, t1 = ds.t0, ds.t1
rows_out: List[Dict[str, Any]] = []
exch_dt = _sql_ws_tick_exchange_dt()
@@ -334,10 +557,10 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
"FROM ("
" SELECT source, "
f" TIMESTAMPDIFF(SECOND, {exch_dt}, recv_ts) AS lag_sec "
" FROM ws_ticks WHERE tick_time LIKE %s"
f" FROM {tick_tbl}{tick_where}"
") t GROUP BY source"
)
for r in db.conn.execute(tick_sql, (age, age, like)).fetchall() or []:
for r in db.conn.execute(tick_sql, (age, age, *tick_params)).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
rows_out.append(
_age_cut_row(
@@ -361,43 +584,57 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
" FROM ("
" SELECT code, tick_time, "
f" TIMESTAMPDIFF(SECOND, {exch_dt}, recv_ts) AS lag_sec "
" FROM ws_ticks WHERE tick_time LIKE %s"
f" FROM {tick_tbl}{tick_where}"
" ) x "
" GROUP BY code, LEFT(tick_time,12) "
" HAVING SUM(CASE WHEN lag_sec IS NULL OR lag_sec <= %s THEN 1 ELSE 0 END)=0 "
" AND COUNT(*)>0"
") z"
)
r = db.conn.execute(empty_sql, (like, age)).fetchone()
r = db.conn.execute(empty_sql, (*tick_params, age)).fetchone()
empty_min = int((r["n"] if isinstance(r, dict) else r[0]) or 0)
except Exception:
empty_min = 0
# ── ls_ws_ticks ──
try:
d0 = datetime.strptime(day8, "%Y%m%d")
d1 = d0.replace(hour=23, minute=59, second=59)
ls_sql = (
"SELECT 'ls' AS src, "
"COUNT(*) AS total, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec <= %s THEN 1 ELSE 0 END) AS pass_n, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec > %s THEN 1 ELSE 0 END) AS fail_n, "
"SUM(CASE WHEN lag_sec IS NULL THEN 1 ELSE 0 END) AS unknown_n "
"FROM ("
" SELECT TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE("
" CASE WHEN CHAR_LENGTH(REGEXP_REPLACE(IFNULL(chetime,''), '[^0-9]', ''))>=14 "
" THEN LEFT(REGEXP_REPLACE(chetime, '[^0-9]', ''), 14) "
" WHEN CHAR_LENGTH(REGEXP_REPLACE(IFNULL(chetime,''), '[^0-9]', ''))>=6 "
" THEN CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), "
" RIGHT(REGEXP_REPLACE(chetime, '[^0-9]', ''), 6)) "
" ELSE DATE_FORMAT(ts,'%%Y%%m%%d%%H%%i%%s') END, "
" '%%Y%%m%%d%%H%%i%%s'), "
" ts) AS lag_sec "
" FROM ls_ws_ticks WHERE ts>=%s AND ts<=%s"
") t"
)
r = db.conn.execute(ls_sql, (age, age, d0, d1)).fetchone()
ls_tbl = ds.ls_tick_from()
if ls_tbl:
# TEMP 사용: lag_sec 이미 계산됨 → REGEXP 연산 없음
ls_sql = (
"SELECT 'ls' AS src, "
"COUNT(*) AS total, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec <= %s THEN 1 ELSE 0 END) AS pass_n, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec > %s THEN 1 ELSE 0 END) AS fail_n, "
"SUM(CASE WHEN lag_sec IS NULL THEN 1 ELSE 0 END) AS unknown_n "
f"FROM {ls_tbl}"
)
r = db.conn.execute(ls_sql, (age, age)).fetchone()
else:
# TEMP 없으면 본 테이블 폴백
d0 = datetime.strptime(day8, "%Y%m%d")
d1 = d0.replace(hour=23, minute=59, second=59)
ls_sql = (
"SELECT 'ls' AS src, "
"COUNT(*) AS total, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec <= %s THEN 1 ELSE 0 END) AS pass_n, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec > %s THEN 1 ELSE 0 END) AS fail_n, "
"SUM(CASE WHEN lag_sec IS NULL THEN 1 ELSE 0 END) AS unknown_n "
"FROM ("
" SELECT IFNULL(lag_sec, TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE("
" CASE WHEN CHAR_LENGTH(REGEXP_REPLACE(IFNULL(chetime,''), '[^0-9]', ''))>=14 "
" THEN LEFT(REGEXP_REPLACE(chetime, '[^0-9]', ''), 14) "
" WHEN CHAR_LENGTH(REGEXP_REPLACE(IFNULL(chetime,''), '[^0-9]', ''))>=6 "
" THEN CONCAT(DATE_FORMAT(ts,'%%Y%%m%%d'), "
" RIGHT(REGEXP_REPLACE(chetime, '[^0-9]', ''), 6)) "
" ELSE DATE_FORMAT(ts,'%%Y%%m%%d%%H%%i%%s') END, "
" '%%Y%%m%%d%%H%%i%%s'), "
" ts)) AS lag_sec "
" FROM ls_ws_ticks WHERE ts>=%s AND ts<=%s"
") t"
)
r = db.conn.execute(ls_sql, (age, age, d0, d1)).fetchone()
if r:
d = dict(r) if not isinstance(r, dict) else r
if int(d.get("total") or 0) > 0:
@@ -428,6 +665,12 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
# ── ws_orderbook 스트림 ──
try:
if ds.ready:
ob_stream_from = f" FROM {ob_tbl} WHERE source<>%s"
ob_stream_params: tuple = ("filter_eval",)
else:
ob_stream_from = f" FROM {ob_tbl}{ob_where} AND source<>%s"
ob_stream_params = (*ob_params, "filter_eval")
ob_sql = (
"SELECT source AS src, "
"COUNT(*) AS total, "
@@ -444,10 +687,12 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
" '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts"
" ) AS lag_sec "
" FROM ws_orderbook WHERE snap_time LIKE %s AND source<>%s"
f"{ob_stream_from}"
") t GROUP BY source"
)
for r in db.conn.execute(ob_sql, (age, age, day8, like, "filter_eval")).fetchall() or []:
for r in db.conn.execute(
ob_sql, (age, age, day8, *ob_stream_params)
).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
rows_out.append(
_age_cut_row(
@@ -464,20 +709,33 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
# LS 호가
try:
ls_ob_sql = (
"SELECT 'ls_uh1' AS src, "
"COUNT(*) AS total, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec <= %s THEN 1 ELSE 0 END) AS pass_n, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec > %s THEN 1 ELSE 0 END) AS fail_n, "
"SUM(CASE WHEN lag_sec IS NULL THEN 1 ELSE 0 END) AS unknown_n "
"FROM ("
" SELECT TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE(LEFT(snap_time,14), '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts) AS lag_sec "
" FROM ls_ws_orderbook WHERE snap_time LIKE %s"
") t"
)
r = db.conn.execute(ls_ob_sql, (age, age, like)).fetchone()
ls_ob_tbl = ds.ls_ob_from()
if ls_ob_tbl:
# TEMP 사용: lag_sec 이미 계산됨
ls_ob_sql = (
"SELECT 'ls_uh1' AS src, "
"COUNT(*) AS total, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec <= %s THEN 1 ELSE 0 END) AS pass_n, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec > %s THEN 1 ELSE 0 END) AS fail_n, "
"SUM(CASE WHEN lag_sec IS NULL THEN 1 ELSE 0 END) AS unknown_n "
f"FROM {ls_ob_tbl}"
)
r = db.conn.execute(ls_ob_sql, (age, age)).fetchone()
else:
ls_ob_sql = (
"SELECT 'ls_uh1' AS src, "
"COUNT(*) AS total, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec <= %s THEN 1 ELSE 0 END) AS pass_n, "
"SUM(CASE WHEN lag_sec IS NOT NULL AND lag_sec > %s THEN 1 ELSE 0 END) AS fail_n, "
"SUM(CASE WHEN lag_sec IS NULL THEN 1 ELSE 0 END) AS unknown_n "
"FROM ("
" SELECT IFNULL(lag_sec, TIMESTAMPDIFF(SECOND, "
" STR_TO_DATE(LEFT(snap_time,14), '%%Y%%m%%d%%H%%i%%s'), "
" recv_ts)) AS lag_sec "
" FROM ls_ws_orderbook WHERE snap_time >= %s AND snap_time <= %s"
") t"
)
r = db.conn.execute(ls_ob_sql, (age, age, t0, t1)).fetchone()
if r:
d = dict(r) if not isinstance(r, dict) else r
rows_out.append(
@@ -532,8 +790,8 @@ def _feed_age_cut_stats(db, day8: str, age_sec: float) -> Dict[str, Any]:
pass
tick_chain = _vendor_read_chain(tick_primary)
ob_chain = _vendor_read_chain(ob_primary)
tick_wins = _count_tick_picks(db, day8, age, tick_chain)
ob_wins = _count_ob_picks(db, day8, age, ob_chain)
tick_wins = _count_tick_picks(db, day8, age, tick_chain, day_slice=ds)
ob_wins = _count_ob_picks(db, day8, age, ob_chain, day_slice=ds)
_annotate_share_and_pick(
rows_out,
tick_wins=tick_wins,
@@ -599,17 +857,35 @@ def _gap_row_from_sql(d: Dict[str, Any], *, soft: int, hard: int) -> Dict[str, A
}
def _feed_disconnect_stats(db, day8: str) -> Dict[str, Any]:
def _feed_disconnect_stats(
db,
day8: str,
day_slice: Optional[_FeedStatsDaySlice] = None,
) -> Dict[str, Any]:
"""종목별 recv_ts 간격으로 끊김(연결 공백) 집계 — usable(체결시각 lag)과 다른 축.
장중(09:00~15:30) · 연속 틱 LAG. cap 초과 공백은 미구독/장외로 제외.
"""
soft, hard, cap = _disconnect_thresholds()
like = _like(day8)
ds = day_slice or _FeedStatsDaySlice(db, day8)
tick_tbl = ds.tick_from()
tick_where, tick_params = ds.tick_filter_sql()
rows: List[Dict[str, Any]] = []
by_vendor: Dict[str, Dict[str, Any]] = {}
try:
if ds.ready:
tick_lag_where = (
f" FROM {tick_tbl} "
"WHERE TIME(recv_ts) BETWEEN '09:00:00' AND '15:30:00'"
)
tick_lag_params: tuple = ()
else:
tick_lag_where = (
f" FROM {tick_tbl}{tick_where} "
"AND TIME(recv_ts) BETWEEN '09:00:00' AND '15:30:00'"
)
tick_lag_params = tick_params
sql = (
"SELECT source AS vendor, "
"COUNT(*) AS pair_n, "
@@ -622,14 +898,14 @@ def _feed_disconnect_stats(db, day8: str) -> Dict[str, Any]:
" TIMESTAMPDIFF(SECOND, "
" LAG(recv_ts) OVER (PARTITION BY source, code ORDER BY recv_ts, id), "
" recv_ts) AS gap_sec "
" FROM ws_ticks "
" WHERE tick_time LIKE %s "
" AND TIME(recv_ts) BETWEEN '09:00:00' AND '15:30:00'"
f"{tick_lag_where}"
") t "
"WHERE gap_sec IS NOT NULL AND gap_sec > 0 AND gap_sec < %s "
"GROUP BY source"
)
for r in db.conn.execute(sql, (soft, hard, soft, like, cap)).fetchall() or []:
for r in db.conn.execute(
sql, (soft, hard, soft, *tick_lag_params, cap)
).fetchall() or []:
d = dict(r) if not isinstance(r, dict) else r
row = _gap_row_from_sql(d, soft=soft, hard=hard)
v = row["vendor"]
@@ -891,57 +1167,124 @@ def build_feed_collect_stats(
day: Optional[str] = None,
*,
heavy: bool = False,
invalidate_cache: bool = False,
) -> Dict[str, Any]:
"""한 거래일(KST YYYYMMDD) 수집 요약 + 증권사 대비.
기본(heavy=False): GROUP BY COUNT + 폴백나이(STR_TO_DATE 전수, 보통 십수 초).
heavy=True: 추가로 DISTINCT 종목교집합·filter_eval pass/reject.
최근 N일 일괄 GROUP BY 는 제거(틱·호가 테이블 폭주).
종료: 결과는 5분 TTL 메모리 캐시에 저장 (웹탭 재조회 시 즉시 반환).
invalidate_cache=True 이면 캐시 무시 및 강제 재계산.
"""
global _STATS_CACHE, _STATS_CACHE_TTL
day8 = _ymd8(day or "")
# 캐시 히트 확인
if not invalidate_cache:
cached = _STATS_CACHE.get(day8)
if cached:
expire_ts, result = cached
if _time.monotonic() < expire_ts:
return result
result = _build_feed_collect_stats_inner(db, day8, heavy=heavy)
# 캐시 저장 (과거일의 경우 1시간, 당일은 5분)
today8 = datetime.now().strftime("%Y%m%d")
ttl = _STATS_CACHE_TTL if day8 >= today8 else 3600.0
_STATS_CACHE[day8] = (_time.monotonic() + ttl, result)
# 낡은 캐시 항목 정리
_STATS_CACHE = {
k: v for k, v in _STATS_CACHE.items()
if _time.monotonic() < v[0]
}
return result
def invalidate_feed_collect_stats_cache(day8: Optional[str] = None) -> None:
"""캐시 무효화. day8 지정 시 해당 일자만, None이면 전체."""
global _STATS_CACHE
if day8:
_STATS_CACHE.pop(day8, None)
else:
_STATS_CACHE.clear()
def _build_feed_collect_stats_inner(
db,
day8: str,
*,
heavy: bool = False,
) -> Dict[str, Any]:
"""build_feed_collect_stats 실제 연산 바디 (캐시 래퍼 제외)."""
t0, t1 = _day_bounds(day8)
like = _like(day8)
notes: List[str] = []
heavy = bool(heavy)
day_slice = _FeedStatsDaySlice(db, day8)
if not day_slice.ensure():
notes.append(
"TEMP 일별 슬라이스 생성 실패 — tick_time range 직접 스캔으로 폴백."
)
elif day_slice.tick_n > 0 or day_slice.ob_n > 0:
notes.append(
f"일별 TEMP 슬라이스: tick={day_slice.tick_n:,} ob={day_slice.ob_n:,} "
"(lag·끊김·pick은 TEMP만 스캔)."
)
tick_tbl = day_slice.tick_from()
ob_tbl = day_slice.ob_from()
tick_where, tick_params = day_slice.tick_filter_sql()
ob_where, ob_params = day_slice.ob_filter_sql()
ticks_by_src = _safe_group(
db,
"SELECT source AS k, COUNT(*) AS n, COUNT(DISTINCT code) AS c "
"FROM ws_ticks WHERE tick_time LIKE %s GROUP BY source ORDER BY n DESC",
(like,),
f"FROM {tick_tbl}{tick_where} GROUP BY source ORDER BY n DESC",
tick_params,
)
ob_by_src = _safe_group(
db,
"SELECT source AS k, COUNT(*) AS n, COUNT(DISTINCT code) AS c "
"FROM ws_orderbook WHERE snap_time LIKE %s GROUP BY source ORDER BY n DESC",
(like,),
f"FROM {ob_tbl}{ob_where} GROUP BY source ORDER BY n DESC",
ob_params,
)
if day_slice.ready:
fe_where = f" FROM {ob_tbl} WHERE source=%s"
fe_params: tuple = ("filter_eval",)
else:
fe_where = f" FROM {ob_tbl}{ob_where} AND source=%s"
fe_params = (*ob_params, "filter_eval")
fe_by_strat = _safe_group(
db,
"SELECT COALESCE(strategy,'') AS k, COUNT(*) AS n, COUNT(DISTINCT code) AS c "
"FROM ws_orderbook WHERE source=%s AND snap_time LIKE %s "
f"{fe_where} "
"GROUP BY strategy ORDER BY n DESC",
("filter_eval", like),
fe_params,
)
fe_reject = {"n": 0, "codes": 0}
fe_pass = {"n": 0, "codes": 0}
if heavy:
fe_reject = _safe_nc(
db,
"SELECT COUNT(*) AS n, COUNT(DISTINCT code) AS c FROM ws_orderbook "
"WHERE source=%s AND snap_time LIKE %s AND reject_code IS NOT NULL AND reject_code<>''",
("filter_eval", like),
"SELECT COUNT(*) AS n, COUNT(DISTINCT code) AS c "
f"{fe_where} AND reject_code IS NOT NULL AND reject_code<>''",
fe_params,
)
fe_pass = _safe_nc(
db,
"SELECT COUNT(*) AS n, COUNT(DISTINCT code) AS c FROM ws_orderbook "
"WHERE source=%s AND snap_time LIKE %s AND (reject_code IS NULL OR reject_code='')",
("filter_eval", like),
"SELECT COUNT(*) AS n, COUNT(DISTINCT code) AS c "
f"{fe_where} AND (reject_code IS NULL OR reject_code='')",
fe_params,
)
ls_ob = _safe_nc(
db,
"SELECT COUNT(*) AS n, COUNT(DISTINCT code) AS c FROM ls_ws_orderbook WHERE snap_time LIKE %s",
(like,),
"SELECT COUNT(*) AS n, COUNT(DISTINCT code) AS c FROM ls_ws_orderbook "
"WHERE snap_time >= %s AND snap_time <= %s",
(t0, t1),
)
try:
d0 = datetime.strptime(day8, "%Y%m%d")
@@ -999,26 +1342,48 @@ def build_feed_collect_stats(
ls_tick_codes: Optional[Set[str]] = None
ls_ob_codes: Optional[Set[str]] = None
if heavy:
kis_tick_codes = _distinct_codes(
db,
"SELECT DISTINCT code FROM ws_ticks WHERE tick_time LIKE %s AND source=%s",
(like, "kis"),
)
kis_ob_codes = _distinct_codes(
db,
"SELECT DISTINCT code FROM ws_orderbook WHERE snap_time LIKE %s AND source=%s",
(like, "kis_h0stasp0"),
)
kw_tick_codes = _distinct_codes(
db,
"SELECT DISTINCT code FROM ws_ticks WHERE tick_time LIKE %s AND source=%s",
(like, "kiwoom"),
)
kw_ob_codes = _distinct_codes(
db,
"SELECT DISTINCT code FROM ws_orderbook WHERE snap_time LIKE %s AND source=%s",
(like, "kiwoom_0d"),
)
if day_slice.ready:
kis_tick_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {tick_tbl} WHERE source=%s",
("kis",),
)
kis_ob_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {ob_tbl} WHERE source=%s",
("kis_h0stasp0",),
)
kw_tick_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {tick_tbl} WHERE source=%s",
("kiwoom",),
)
kw_ob_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {ob_tbl} WHERE source=%s",
("kiwoom_0d",),
)
else:
kis_tick_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {tick_tbl}{tick_where} AND source=%s",
(*tick_params, "kis"),
)
kis_ob_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {ob_tbl}{ob_where} AND source=%s",
(*ob_params, "kis_h0stasp0"),
)
kw_tick_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {tick_tbl}{tick_where} AND source=%s",
(*tick_params, "kiwoom"),
)
kw_ob_codes = _distinct_codes(
db,
f"SELECT DISTINCT code FROM {ob_tbl}{ob_where} AND source=%s",
(*ob_params, "kiwoom_0d"),
)
try:
d0 = datetime.strptime(day8, "%Y%m%d")
d1 = d0.replace(hour=23, minute=59, second=59)
@@ -1031,8 +1396,8 @@ def build_feed_collect_stats(
ls_tick_codes = set()
ls_ob_codes = _distinct_codes(
db,
"SELECT DISTINCT code FROM ls_ws_orderbook WHERE snap_time LIKE %s",
(like,),
"SELECT DISTINCT code FROM ls_ws_orderbook WHERE snap_time >= %s AND snap_time <= %s",
(t0, t1),
)
coverage = [
@@ -1173,8 +1538,8 @@ def build_feed_collect_stats(
notes.append("LS_WS_TICK_SAVE=true → 구독 종목 US3를 ls_ws_ticks 에 적재 (영구만이 아님).")
# 폴백나이: 기본 조회에도 포함 (실측 ~10초대, 요약테이블/크론 불필요)
age_cut = _feed_age_cut_stats(db, day8, fb_age)
disconnect = _feed_disconnect_stats(db, day8)
age_cut = _feed_age_cut_stats(db, day8, fb_age, day_slice=day_slice)
disconnect = _feed_disconnect_stats(db, day8, day_slice=day_slice)
return {
"day": f"{day8[:4]}-{day8[4:6]}-{day8[6:8]}",