0780b2cdd0
Changes: - Added new API endpoints for continuing and confirming Optuna jobs, allowing for better management of ongoing studies. - Introduced detailed logging for tick feed tracking and order book processing, improving traceability of vendor performance during backtests. - Updated database schema to include new fields for managing Optuna study results, enhancing the ability to track study progress and outcomes. - Refactored existing functions to utilize the new logging and tracking features, ensuring consistency across the backtesting framework. Impact: - These enhancements improve the robustness and transparency of the Optuna backtesting process, facilitating better analysis and optimization of trading strategies.
251 lines
8.5 KiB
Python
251 lines
8.5 KiB
Python
"""
|
|
ws_candles 시리즈 — source=증권사, channel=ws|rest|rollup.
|
|
|
|
저장은 섞지 않음: 키움 REST 갭보정은 항상 source=kiwoom, channel=rest.
|
|
읽기(실매=Optuna): 메인 WS → 2차 WS → LS WS → kiwoom+rest → rollup.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
|
|
|
|
from kis_trader.utils.env import get_env_from_db
|
|
|
|
CHANNEL_WS = "ws"
|
|
CHANNEL_REST = "rest"
|
|
CHANNEL_ROLLUP = "rollup"
|
|
|
|
# 읽기 우선순위 쌍 (source, channel)
|
|
ReadPair = Tuple[str, str]
|
|
|
|
|
|
def normalize_source_channel(
|
|
source: str = "",
|
|
channel: str = "",
|
|
) -> Tuple[str, str]:
|
|
"""레거시 source=rest|kw_rest|rollup_1m 을 (증권사, channel)로 정규화."""
|
|
s = str(source or "").strip().lower()
|
|
ch = str(channel or "").strip().lower()
|
|
if s in ("rest", "kw_rest"):
|
|
return "kiwoom", CHANNEL_REST
|
|
if s in ("rollup_1m", "rollup"):
|
|
src = "kiwoom"
|
|
if ch in ("kis",):
|
|
src = "kis"
|
|
return src, CHANNEL_ROLLUP
|
|
if s in ("kis", "kis_us", "ws", ""):
|
|
return "kis", (ch or CHANNEL_WS)
|
|
if s == "kiwoom":
|
|
return "kiwoom", (ch or CHANNEL_WS)
|
|
if s == "ls":
|
|
return "ls", (ch or CHANNEL_WS)
|
|
return (s or "kis")[:16], (ch or CHANNEL_WS)[:10]
|
|
|
|
|
|
def _provider_and_override() -> Tuple[str, str]:
|
|
override = ""
|
|
raw = os.environ.get("CANDLE_SOURCE")
|
|
if raw is None:
|
|
try:
|
|
raw = get_env_from_db("CANDLE_SOURCE", "")
|
|
except Exception:
|
|
raw = ""
|
|
override = str(raw or "").strip().lower()
|
|
if override not in ("kis", "kiwoom"):
|
|
override = ""
|
|
try:
|
|
provider = (
|
|
get_env_from_db("LIVE_TICK_PROVIDER", "kiwoom") or "kiwoom"
|
|
).strip().lower()
|
|
except Exception:
|
|
provider = "kiwoom"
|
|
if provider not in ("kis", "kiwoom"):
|
|
provider = "kiwoom"
|
|
return provider, override
|
|
|
|
|
|
def live_read_pairs() -> Tuple[ReadPair, ...]:
|
|
"""실매·Optuna 기본 읽기 순서.
|
|
|
|
메인 WS → 2차 WS → LS WS(있으면) → 키움 REST 구멍 → 롤업.
|
|
CANDLE_SOURCE=kis|kiwoom 이면 그 메인을 씀. 빈값이면 LIVE_TICK_PROVIDER.
|
|
"""
|
|
provider, override = _provider_and_override()
|
|
main = override or provider
|
|
aux = "kis" if main == "kiwoom" else "kiwoom"
|
|
return (
|
|
(main, CHANNEL_WS),
|
|
(aux, CHANNEL_WS),
|
|
("ls", CHANNEL_WS),
|
|
("kiwoom", CHANNEL_REST),
|
|
("kiwoom", CHANNEL_ROLLUP),
|
|
)
|
|
|
|
|
|
def live_read_label() -> str:
|
|
provider, override = _provider_and_override()
|
|
main = override or provider
|
|
aux = "kis" if main == "kiwoom" else "kiwoom"
|
|
if override:
|
|
return f"{override.upper()}+{aux}+ls+kiwoom_rest"
|
|
return f"LIVE({main})+{aux}+ls+kiwoom_rest"
|
|
|
|
|
|
def _index_ticks_by_minute(ticks: Sequence[Dict[str, Any]]) -> Dict[Tuple[str, str], List[Dict[str, Any]]]:
|
|
"""(source, YYYYMMDDHHMM) → 틱. source 빈 틱은 ('*', 분) — 모든 증권사 쓰레기검사에 포함."""
|
|
idx: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
|
|
for t in ticks or []:
|
|
if not isinstance(t, dict):
|
|
continue
|
|
raw = str(t.get("tick_time_raw") or t.get("tick_time") or "").strip()
|
|
digits = raw.replace(":", "").replace("-", "").replace(" ", "")
|
|
tmin = digits[:12] if len(digits) >= 12 else ""
|
|
if len(tmin) < 12 or not tmin.isdigit():
|
|
continue
|
|
tsrc = str(t.get("source") or "").strip().lower()
|
|
key = (tsrc if tsrc else "*", tmin)
|
|
idx.setdefault(key, []).append(t)
|
|
return idx
|
|
|
|
|
|
def _ticks_in_bar_from_index(
|
|
idx: Optional[Dict[Tuple[str, str], List[Dict[str, Any]]]],
|
|
source: str,
|
|
candle_time: str,
|
|
tf_min: int,
|
|
) -> List[Dict[str, Any]]:
|
|
"""tick_in_bar_bucket 과 같은 분 구간만 — bar_is_garbage 전체 스캔 방지."""
|
|
from kis_trader.engine.candle_rollup import add_candle_minutes
|
|
|
|
if not idx:
|
|
return []
|
|
ct = str(candle_time or "").strip()[:12]
|
|
if len(ct) < 12:
|
|
return []
|
|
end = add_candle_minutes(ct, max(1, int(tf_min or 1)))
|
|
if not end:
|
|
return []
|
|
end12 = end[:12]
|
|
src = str(source or "").strip().lower()
|
|
out: List[Dict[str, Any]] = []
|
|
cur = ct
|
|
while cur and cur < end12:
|
|
if src:
|
|
out.extend(idx.get((src, cur), ()))
|
|
out.extend(idx.get(("*", cur), ()))
|
|
nxt = add_candle_minutes(cur, 1)
|
|
if not nxt or nxt[:12] <= cur:
|
|
break
|
|
cur = nxt[:12]
|
|
return out
|
|
|
|
|
|
def dedupe_by_read_pairs(
|
|
rows: Sequence[Dict[str, Any]],
|
|
pairs: Optional[Sequence[ReadPair]] = None,
|
|
*,
|
|
ticks: Optional[Sequence[Dict[str, Any]]] = None,
|
|
tf_min: int = 1,
|
|
missing_policy: str = "hole",
|
|
live_cover: bool = False,
|
|
stats: Optional[Dict[str, Any]] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
"""candle_time 1행 — pairs 앞쪽이 이김. source/channel 은 결과에 남기지 않음.
|
|
|
|
ticks 가 있으면 WS 봉만 쓰레기(0건·봉끝 나이초과)일 때 다음 쌍. 한 봉 혼합 없음.
|
|
live_cover=True 이면 링이 그 분을 커버 못할 때 그 WS 봉을 유지.
|
|
stats 가 있으면 pick/garbage/hole 누적 (옵투나 로그용).
|
|
"""
|
|
from kis_trader.engine.feed_fallback import bar_is_garbage, live_bar_is_garbage
|
|
|
|
order = tuple(pairs or live_read_pairs())
|
|
by_ct: Dict[str, Dict[ReadPair, Dict[str, Any]]] = {}
|
|
for row in rows:
|
|
ct = str(row.get("candle_time") or "")[:12]
|
|
if not ct:
|
|
continue
|
|
src, ch = normalize_source_channel(
|
|
str(row.get("source") or ""),
|
|
str(row.get("channel") or ""),
|
|
)
|
|
by_ct.setdefault(ct, {})
|
|
pair = (src, ch)
|
|
if pair not in by_ct[ct]:
|
|
by_ct[ct][pair] = dict(row)
|
|
|
|
tf = max(1, int(tf_min or 1))
|
|
tick_idx = None
|
|
if ticks is not None:
|
|
tick_idx = _index_ticks_by_minute(ticks)
|
|
if stats is not None:
|
|
stats.setdefault("slots", 0)
|
|
stats.setdefault("picked", 0)
|
|
stats.setdefault("hole", 0)
|
|
stats.setdefault("garbage_skip", 0)
|
|
stats.setdefault("pick_by", {})
|
|
stats.setdefault("garbage_by", {})
|
|
stats.setdefault("raw_by", {})
|
|
for ct_map in by_ct.values():
|
|
for pair in ct_map:
|
|
key = "%s/%s" % pair
|
|
stats["raw_by"][key] = int(stats["raw_by"].get(key) or 0) + 1
|
|
out: List[Dict[str, Any]] = []
|
|
for ct in sorted(by_ct.keys()):
|
|
if stats is not None:
|
|
stats["slots"] = int(stats.get("slots") or 0) + 1
|
|
picked: Optional[Dict[str, Any]] = None
|
|
pick_key = ""
|
|
for pair in order:
|
|
bar = by_ct[ct].get(pair)
|
|
if bar is None:
|
|
continue
|
|
src, ch = pair
|
|
if ticks is not None and ch == CHANNEL_WS:
|
|
scoped = _ticks_in_bar_from_index(tick_idx, src, ct, tf)
|
|
if live_cover:
|
|
bad = live_bar_is_garbage(
|
|
ticks, candle_time=ct, tf_min=tf, source=src,
|
|
)
|
|
else:
|
|
bad = bar_is_garbage(
|
|
scoped, candle_time=ct, tf_min=tf, source=src,
|
|
missing_policy=missing_policy,
|
|
)
|
|
if bad:
|
|
if stats is not None:
|
|
stats["garbage_skip"] = int(stats.get("garbage_skip") or 0) + 1
|
|
gk = "%s/%s" % pair
|
|
stats["garbage_by"][gk] = int(stats["garbage_by"].get(gk) or 0) + 1
|
|
continue
|
|
picked = dict(bar)
|
|
pick_key = "%s/%s" % pair
|
|
break
|
|
if picked is None:
|
|
if stats is not None:
|
|
stats["hole"] = int(stats.get("hole") or 0) + 1
|
|
continue
|
|
if stats is not None:
|
|
stats["picked"] = int(stats.get("picked") or 0) + 1
|
|
stats["pick_by"][pick_key] = int(stats["pick_by"].get(pick_key) or 0) + 1
|
|
picked.pop("source", None)
|
|
picked.pop("channel", None)
|
|
out.append(picked)
|
|
return out
|
|
|
|
|
|
def pick_bar_by_read_pairs(
|
|
rows: Sequence[Dict[str, Any]],
|
|
pairs: Optional[Sequence[ReadPair]] = None,
|
|
*,
|
|
ticks: Optional[Sequence[Dict[str, Any]]] = None,
|
|
tf_min: int = 1,
|
|
missing_policy: str = "hole",
|
|
live_cover: bool = False,
|
|
) -> Optional[Dict[str, Any]]:
|
|
"""한 candle_time 묶음에서 승자 봉 1개. 없으면 None."""
|
|
out = dedupe_by_read_pairs(
|
|
rows, pairs, ticks=ticks, tf_min=tf_min,
|
|
missing_policy=missing_policy, live_cover=live_cover,
|
|
)
|
|
return out[0] if out else None
|