hoon/kis_bot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ls증권 히스토리 구독 넣음

Browse Source
This commit is contained in:
Your Name
2026-07-30 18:05:07 +09:00
parent 61bec4bd1d
commit 67eab24603
1593 changed files with 135733 additions and 1232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
kis_trader/backtest/momentum_backtest_common.py
View File

@@ -120,6 +120,8 @@ def prepend_momentum_candle_warmup(
# 종목×기간일 단위 REST 웜업 캐시 (프로세스 메모리만 — DB 미기록)
_REST_WARMUP_PREFIX_CACHE: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
# 1차+재시도 후에도 전일 장시작 시가 미확보 → trial마다 재조회·로그 금지
_REST_WARMUP_PERM_FAIL: Set[Tuple[str, str]] = set()
def _momentum_rows_have_prev_day(rows: List[Dict], period_day: str) -> bool:
@@ -145,6 +147,44 @@ def _kiwoom_gap_credentials() -> Tuple[str, str, bool]:
return key, secret, is_mock
def _rest_df_to_prefix(
df: Any,
rows: List[Dict],
period_start_key: str,
) -> List[Dict[str, Any]]:
"""ka10080 DF → 기간 시작 이전 prefix 봉 리스트."""
ps = str(period_start_key or "")[:12]
first_ct = ""
for r in rows:
ct = str(r.get("candle_time") or "")
if ct >= ps:
first_ct = ct[:12]
break
if not first_ct:
first_ct = ps
existing = {str(r.get("candle_time") or "")[:12] for r in rows}
prefix: List[Dict[str, Any]] = []
for _, rec in df.iterrows():
t = str(rec.get("time") or "")[:12]
if len(t) < 12 or t >= first_ct or t in existing:
continue
op = float(rec.get("open") or 0)
if op <= 0:
continue
prefix.append({
"candle_time": t,
"open": op,
"high": float(rec.get("high") or op),
"low": float(rec.get("low") or op),
"close": float(rec.get("close") or op),
"volume": int(float(rec.get("volume") or 0)),
"is_confirmed": 1,
"_rest_warmup": 1,
})
prefix.sort(key=lambda x: str(x.get("candle_time") or ""))
return prefix
def inject_momentum_rest_warmup_memory(
candles_by_code: Dict[str, List[Dict]],
period_start_key: str,
@@ -152,17 +192,20 @@ def inject_momentum_rest_warmup_memory(
universe_by_slot: Optional[Dict[str, List[str]]] = None,
) -> Dict[str, int]:
"""
DB 전일봉이 없을 때 키움 ka10080 REST를 종목당 1회 호출해 **메모리에만** prepend.
DB 전일봉이 없을 때 키움 ka10080 REST를 종목당 호출해 **메모리에만** prepend.
- 실매: 유니버스 편입 → 갭보정 RAM (DB 구데이터 미사용)
- 백테/Optuna: DB warmup 실패 시 동일 REST 1회로 E·지표 워밍업
- DB INSERT 없음. 프로세스 캐시로 웹 재호출·Optuna trial 재조회 방지.
- 1차: MOMENTUM_BACKTEST_REST_WARMUP_BARS (기본 700)
- 전일 장시작 시가 미확보 시에만 2차: MOMENTUM_BACKTEST_REST_WARMUP_BARS_RETRY (기본 1500)
- DB INSERT 없음. 성공 prefix·영구실패는 프로세스 캐시 (Optuna trial 재조회·로그 스팸 방지).
"""
from kis_trader.utils.env import get_env_bool, get_env_float, get_env_int
from kis_trader.utils.logger import get_logger
log = get_logger("kis_trader.momentum_backtest")
stats = {"need": 0, "ok": 0, "fail": 0, "cache_hit": 0, "bars": 0, "skipped": 0}
stats = {
"need": 0, "ok": 0, "fail": 0, "cache_hit": 0,
"bars": 0, "skipped": 0, "retry": 0,
}
if not get_env_bool("MOMENTUM_BACKTEST_REST_WARMUP", True):
stats["skipped"] = 1
return stats
@@ -184,7 +227,8 @@ def inject_momentum_rest_warmup_memory(
need_codes = [
c for c in sorted(target)
if not _momentum_rows_have_prev_day(candles_by_code.get(c) or [], period_day)
if (c, period_day) not in _REST_WARMUP_PERM_FAIL
and not _momentum_rows_have_prev_day(candles_by_code.get(c) or [], period_day)
]
stats["need"] = len(need_codes)
if not need_codes:
@@ -198,6 +242,11 @@ def inject_momentum_rest_warmup_memory(
50,
int(get_env_int("MOMENTUM_BACKTEST_REST_WARMUP_BARS", 700)),
)
# 전일(직전 세션) 장시작이 1차에 안 잡힐 때만 — 평소엔 700만
n_retry = max(
n_bars,
int(get_env_int("MOMENTUM_BACKTEST_REST_WARMUP_BARS_RETRY", 1500)),
)
sleep_sec = float(get_env_float("MOMENTUM_BACKTEST_REST_SLEEP_SEC", 0.25))
kw_key, kw_secret, is_mock = _kiwoom_gap_credentials()
if not kw_key or not kw_secret:
@@ -207,83 +256,90 @@ def inject_momentum_rest_warmup_memory(
from kis_trader.ws.kis_ws import get_kiwoom_candles_df
log.info(
"📡 모멘텀 REST 웜업(메모리): 전일봉 부족 %d종목 · 종목당 1회 ka10080 n=%d (DB 미기록)",
len(need_codes), n_bars,
# 전부 캐시 hit면 Optuna trial 경로에서 INFO 스팸 금지
will_fetch = any(
(c, period_day) not in _REST_WARMUP_PREFIX_CACHE for c in need_codes
)
if will_fetch:
log.info(
"📡 모멘텀 REST 웜업(메모리): 전일봉 부족 %d종목 · ka10080 n=%d"
" (실패 시 n=%d 1회 재시도, DB 미기록)",
len(need_codes), n_bars, n_retry,
)
for i, code in enumerate(need_codes):
rows = candles_by_code.get(code) or []
if not rows:
_REST_WARMUP_PERM_FAIL.add((code, period_day))
stats["fail"] += 1
continue
cache_key = (code, period_day)
cached = _REST_WARMUP_PREFIX_CACHE.get(cache_key)
did_network = False
if cached is not None:
stats["cache_hit"] += 1
prefix = [dict(r) for r in cached]
else:
did_network = True
try:
df = get_kiwoom_candles_df(
code, 1, kw_key, kw_secret, is_mock=is_mock, n=n_bars,
)
except Exception as e:
log.warning("⚠️ REST 웜업 실패 %s: %s", code, e)
_REST_WARMUP_PERM_FAIL.add(cache_key)
stats["fail"] += 1
continue
if df is None or getattr(df, "empty", True):
_REST_WARMUP_PERM_FAIL.add(cache_key)
stats["fail"] += 1
continue
first_ct = ""
for r in rows:
ct = str(r.get("candle_time") or "")
if ct >= ps:
first_ct = ct[:12]
break
if not first_ct:
first_ct = ps
existing = {str(r.get("candle_time") or "")[:12] for r in rows}
prefix = []
try:
for _, rec in df.iterrows():
t = str(rec.get("time") or "")[:12]
if len(t) < 12 or t >= first_ct or t in existing:
continue
op = float(rec.get("open") or 0)
if op <= 0:
continue
prefix.append({
"candle_time": t,
"open": op,
"high": float(rec.get("high") or op),
"low": float(rec.get("low") or op),
"close": float(rec.get("close") or op),
"volume": int(float(rec.get("volume") or 0)),
"is_confirmed": 1,
"_rest_warmup": 1,
})
prefix = _rest_df_to_prefix(df, rows, ps)
except Exception as e:
log.warning("⚠️ REST 웜업 파싱 실패 %s: %s", code, e)
_REST_WARMUP_PERM_FAIL.add(cache_key)
stats["fail"] += 1
continue
prefix.sort(key=lambda x: str(x.get("candle_time") or ""))
# 1차로 전일 장시작 미확보 → 봉 수 늘려 1회만 재시도 (중간 거래일 0봉 등)
if (not prefix or not _momentum_rows_have_prev_day(prefix, period_day)) and n_retry > n_bars:
stats["retry"] += 1
log.info(
"📡 REST 웜업 재시도 %s: n=%d → n=%d (전일 장시작 미확보)",
code, n_bars, n_retry,
)
try:
df2 = get_kiwoom_candles_df(
code, 1, kw_key, kw_secret, is_mock=is_mock, n=n_retry,
)
except Exception as e:
log.warning("⚠️ REST 웜업 재시도 실패 %s: %s", code, e)
df2 = None
if df2 is not None and not getattr(df2, "empty", True):
try:
prefix = _rest_df_to_prefix(df2, rows, ps)
except Exception as e:
log.warning("⚠️ REST 웜업 재시도 파싱 실패 %s: %s", code, e)
prefix = []
_REST_WARMUP_PREFIX_CACHE[cache_key] = [dict(r) for r in prefix]
if sleep_sec > 0 and i + 1 < len(need_codes):
if sleep_sec > 0 and did_network and i + 1 < len(need_codes):
time.sleep(sleep_sec)
if not prefix:
stats["fail"] += 1
continue
if not _momentum_rows_have_prev_day(prefix, period_day):
if not prefix or not _momentum_rows_have_prev_day(prefix, period_day):
_REST_WARMUP_PERM_FAIL.add(cache_key)
stats["fail"] += 1
continue
candles_by_code[code] = [dict(r) for r in prefix] + [dict(r) for r in rows]
stats["ok"] += 1
stats["bars"] += len(prefix)
log.info(
"✅ 모멘텀 REST 웜업 완료: ok=%d fail=%d cache=%d bars=%d",
stats["ok"], stats["fail"], stats["cache_hit"], stats["bars"],
)
if will_fetch:
log.info(
"✅ 모멘텀 REST 웜업 완료: ok=%d fail=%d cache=%d retry=%d bars=%d",
stats["ok"], stats["fail"], stats["cache_hit"], stats["retry"], stats["bars"],
)
return stats
@@ -299,6 +355,7 @@ def resolve_momentum_universe(
*,
use_saved_history: bool,
strategy_id: str = MOMENTUM_STRATEGY_ID,
history_source: str = "kiwoom",
) -> Tuple[Optional[Dict[str, List[str]]], str, int, int, str]:
"""
Returns:
@@ -308,9 +365,14 @@ def resolve_momentum_universe(
if use_saved_history and strategy_id:
try:
from kis_trader.database.db_manager import get_db as _get_ext_db
from kis_trader.backtest.universe_history_source import (
history_source_label,
resolve_backtest_universe_history_source,
)
strict = momentum_backtest_universe_strict_enabled()
lag_min = momentum_backtest_universe_strict_lag_min()
debounce_sec = momentum_universe_exit_debounce_sec()
hs = resolve_backtest_universe_history_source(history_source)
history = _get_ext_db().get_universe_by_candle_time(
strategy_id=strategy_id,
start_ymd=start_ymd,
@@ -318,10 +380,11 @@ def resolve_momentum_universe(
strict=strict,
strict_lag_minutes=lag_min,
exit_debounce_sec=debounce_sec,
history_source=hs,
)
if history:
timing = "strict" if strict else "minute"
label = "history_strict" if strict else "history"
label = history_source_label(hs, strict=strict)
return history, label, len(history), 1, timing
except Exception:
pass
@@ -334,25 +397,46 @@ def load_momentum_candles_by_code(
end_key: str,
*,
warmup_bars: Optional[int] = None,
market: Optional[str] = None,
) -> Tuple[Dict[str, List[Dict]], int]:
"""
market: None/빈값 = 전체(기존 동작), 'US'|'KR' = ws_candles.market 필터.
"""
period_start = str(start_key)[:12]
codes_raw = db.conn.execute(
"SELECT DISTINCT code FROM ws_candles WHERE timeframe=1 "
"AND candle_time >= %s AND candle_time <= %s ORDER BY code",
[start_key, end_key],
).fetchall()
mk = (market or "").strip().upper()
if mk:
codes_raw = db.conn.execute(
"SELECT DISTINCT code FROM ws_candles WHERE timeframe=1 AND market=%s "
"AND candle_time >= %s AND candle_time <= %s ORDER BY code",
[mk, start_key, end_key],
).fetchall()
else:
codes_raw = db.conn.execute(
"SELECT DISTINCT code FROM ws_candles WHERE timeframe=1 "
"AND candle_time >= %s AND candle_time <= %s ORDER BY code",
[start_key, end_key],
).fetchall()
codes = [r["code"] for r in codes_raw]
ind_cols = ws_candles_select_indicator_cols(db)
candles_by_code: Dict[str, List[Dict]] = {}
total = 0
for code in codes:
rows = db.conn.execute(
f"SELECT candle_time, open, high, low, close, volume, is_confirmed{ind_cols} "
"FROM ws_candles WHERE timeframe=1 AND code=%s "
"AND candle_time >= %s AND candle_time <= %s "
"ORDER BY candle_time ASC",
[code, start_key, end_key],
).fetchall()
if mk:
rows = db.conn.execute(
f"SELECT candle_time, open, high, low, close, volume, is_confirmed{ind_cols} "
"FROM ws_candles WHERE timeframe=1 AND code=%s AND market=%s "
"AND candle_time >= %s AND candle_time <= %s "
"ORDER BY candle_time ASC",
[code, mk, start_key, end_key],
).fetchall()
else:
rows = db.conn.execute(
f"SELECT candle_time, open, high, low, close, volume, is_confirmed{ind_cols} "
"FROM ws_candles WHERE timeframe=1 AND code=%s "
"AND candle_time >= %s AND candle_time <= %s "
"ORDER BY candle_time ASC",
[code, start_key, end_key],
).fetchall()
if len(rows) < 6:
continue
candles_by_code[code] = [dict(r) for r in rows]
@@ -414,14 +498,19 @@ def run_momentum_backtest_web_aligned(
if not period_start_key:
period_start_key = str(p.get("_backtest_period_start_key") or "")[:12]
if len(period_start_key) >= 8:
rest_warmup_stats = inject_momentum_rest_warmup_memory(
candles_by_code,
period_start_key,
universe_by_slot=universe_by_slot,
)
# 해외 US 티커는 키움 분봉 REST 불가 — 유량 낭비·실패 폭주 방지
if str(p.get("market") or "").strip().upper() != "US":
rest_warmup_stats = inject_momentum_rest_warmup_memory(
candles_by_code,
period_start_key,
universe_by_slot=universe_by_slot,
)
p["slot_money"] = float(slot_money)
p["fee_rate"] = float(fee_rate)
p["sell_tax"] = float(sell_tax)
# 해외 환전: params 에 있으면 유지 (Optuna base_fixed / 웹 US params)
if "fx_fee_rate" not in p:
p["fx_fee_rate"] = 0.0
p["max_stocks"] = int(max_stocks)
if total_budget_krw > 0:
p["total_budget_krw"] = float(total_budget_krw)
@@ -442,17 +531,26 @@ def run_momentum_backtest_web_aligned(
start_key = str(meta_out.get("start_key") or "")
end_key = str(meta_out.get("end_key") or "")
db = meta_out.get("db")
if db is None and start_key and end_key:
from kis_trader.backtest.backtest_portfolio_common import ensure_meta_db
db = ensure_meta_db(meta_out)
if db and start_key and end_key:
_mkt = str(p.get("market") or "KR").strip().upper() or "KR"
loaded_ticks, tick_rows = load_momentum_ticks_by_code(
db, start_key, end_key, set(candles_by_code.keys()),
market=_mkt,
)
tick_meta = tick_coverage_stats(candles_by_code, loaded_ticks)
tick_meta["ws_tick_rows_loaded"] = tick_rows
tick_meta["ws_ticks_table"] = "ws_ticks_us" if _mkt == "US" else "ws_ticks"
if tick_rows <= 0:
from kis_trader.utils.logger import get_logger as _get_logger
_tick_tbl = tick_meta["ws_ticks_table"]
_get_logger("kis_trader.momentum_backtest").warning(
"⚠️ ws_ticks 데이터 없음 — 1분봉 OHLC 청산 폴백 (틱 수집 후 재백테 권장)",
"⚠️ %s 데이터 없음 — 1분봉 OHLC 청산 폴백 (틱 수집 후 재백테 권장)",
_tick_tbl,
)
elif loaded_ticks:
tick_meta = tick_coverage_stats(candles_by_code, loaded_ticks)
@@ -512,9 +610,13 @@ def run_momentum_backtest_web_aligned(
program_by_code=pg_loaded,
)
if not p.get("portfolio_mode"):
_fx = float(p.get("fx_fee_rate", 0.0) or 0.0)
_us = str(p.get("market") or "").strip().upper() == "US"
attach_scalp_trade_pnl(
trades, fee_rate=fee_rate, sell_tax=sell_tax,
slip_pct=backtest_slip_pct(p),
fx_fee_rate=_fx,
pnl_decimals=4 if (_us or _fx > 0) else 0,
)
if meta_out is not None:
meta_out["skip_stats"] = p.get("_portfolio_skip_stats") or {}