36a3e2b4a1
Changes: - Added a new API endpoint for managing permanent subscriptions, allowing users to enable or disable subscriptions dynamically. - Implemented a function to fill candle data from Kiwoom, ensuring that only relevant data is inserted into the database. - Introduced a mechanism to handle master subscription states, improving the management of subscription statuses. - Updated the database schema to include new fields for managing subscription states and order book filtering. Impact: - These enhancements improve the flexibility and reliability of the trading system, allowing for better management of subscriptions and order book data, while reducing the risk of data inconsistencies. 히스토리 align 제거 븅신같은 초기설계 아예 제거 진입모드에 구멍메움 호가진입을 켜도 호가가 안들어올때 호가 안보고 그냥 사버림
449 lines
15 KiB
Python
449 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
매수~매도(또는 ~now) 구간 1분봉 REST 백필 — 판 뒤 1회 / 보유 중 즉시 백필.
|
||
|
||
실매는 벽시계로 청산하지만, ws_candles 가 중간에 끊기면 백테가
|
||
그 종목 포지션을 오후까지 붙잡아 슬롯이 막힌다.
|
||
→ 보유 구간의 분봉을 키움 ka10080 으로 채운다 (DB UPSERT).
|
||
|
||
- 기본 OFF 아님: ``POST_SELL_CANDLE_BACKFILL`` 기본 true
|
||
- 매도 체결 후 백그라운드 1회 (주문 경로 비차단)
|
||
- CLI/스크립트로 과거·현재 보유분 즉시 채우기
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import random
|
||
import threading
|
||
import time
|
||
from datetime import datetime
|
||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||
|
||
from kis_trader.utils.env import get_env_bool, get_env_float, get_env_int
|
||
from kis_trader.utils.logger import get_logger
|
||
|
||
logger = get_logger("kis_trader.post_sell_candle_backfill")
|
||
|
||
# 존재 행 OHLCV 덮어쓰기(큰 volume 우선) — freeze OFF 일 때만
|
||
_INSERT_SQL_OVERWRITE = """
|
||
INSERT INTO ws_candles
|
||
(code, timeframe, candle_time, `open`, high, low, close,
|
||
volume, rsi_2, rsi_3, rsi_5, is_confirmed, source, channel, updated_at)
|
||
VALUES
|
||
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON DUPLICATE KEY UPDATE
|
||
`open`=VALUES(`open`), high=VALUES(high), low=VALUES(low),
|
||
close=VALUES(close),
|
||
volume=IF(VALUES(volume) > volume, VALUES(volume), volume),
|
||
is_confirmed=1, updated_at=VALUES(updated_at),
|
||
source=IF(VALUES(volume) > volume, VALUES(source), source),
|
||
channel=IF(VALUES(volume) > volume, VALUES(channel), channel)
|
||
"""
|
||
|
||
# freeze ON: 없는 분만 INSERT. 확정·미확정 행이 있으면 OHLCV 유지 (구멍 메우기 전용)
|
||
_INSERT_SQL_FREEZE = """
|
||
INSERT INTO ws_candles
|
||
(code, timeframe, candle_time, `open`, high, low, close,
|
||
volume, rsi_2, rsi_3, rsi_5, is_confirmed, source, channel, updated_at)
|
||
VALUES
|
||
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON DUPLICATE KEY UPDATE
|
||
candle_time=candle_time
|
||
"""
|
||
|
||
|
||
def post_sell_candle_backfill_enabled() -> bool:
|
||
return get_env_bool("POST_SELL_CANDLE_BACKFILL", True)
|
||
|
||
|
||
def _candle_freeze_on_confirm() -> bool:
|
||
"""docs/정합성.md — 확정 후 REST/백필이 봉을 키우지 않음. 기본 true."""
|
||
return get_env_bool("WS_CANDLE_FREEZE_ON_CONFIRM", True)
|
||
|
||
|
||
def _insert_sql() -> str:
|
||
return _INSERT_SQL_FREEZE if _candle_freeze_on_confirm() else _INSERT_SQL_OVERWRITE
|
||
|
||
|
||
def post_sell_candle_rollup_3m_enabled() -> bool:
|
||
"""1분 채운 뒤 3분 롤업 UPSERT (꼬리 TF 정합)."""
|
||
return get_env_bool("POST_SELL_CANDLE_ROLLUP_3M", True)
|
||
|
||
|
||
def _dt_to_candle_key(raw: Any) -> str:
|
||
"""datetime / 'YYYY-MM-DD HH:MM:SS' / 'YYYYMMDDHHMM' → YYYYMMDDHHMM."""
|
||
if raw is None:
|
||
return ""
|
||
if isinstance(raw, datetime):
|
||
return raw.strftime("%Y%m%d%H%M")
|
||
s = str(raw).strip()
|
||
if len(s) >= 12 and s[:12].isdigit():
|
||
return s[:12]
|
||
try:
|
||
return datetime.strptime(s[:19], "%Y-%m-%d %H:%M:%S").strftime("%Y%m%d%H%M")
|
||
except ValueError:
|
||
try:
|
||
return datetime.strptime(s[:16], "%Y-%m-%d %H:%M").strftime("%Y%m%d%H%M")
|
||
except ValueError:
|
||
return ""
|
||
|
||
|
||
def _bars_needed(start_key: str, end_key: str) -> int:
|
||
"""
|
||
ka10080 은 **최신→과거** N봉을 주므로, 보유 구간 길이만 요청하면
|
||
장초·어제 보유분은 아예 안 내려온다.
|
||
→ ``지금(또는 end)에서 start 까지`` 캘린더 일수 × 장중분 여유로 요청.
|
||
"""
|
||
cap = max(120, int(get_env_int("POST_SELL_CANDLE_MAX_BARS", 1200)))
|
||
try:
|
||
start = datetime.strptime(start_key[:12], "%Y%m%d%H%M")
|
||
end = datetime.strptime(end_key[:12], "%Y%m%d%H%M")
|
||
except ValueError:
|
||
return min(cap, 500)
|
||
if end < start:
|
||
start, end = end, start
|
||
# REST 기준점 = max(end, now) — 이미 지난 청산도 ‘지금’에서 거슬러 올라감
|
||
now = datetime.now()
|
||
anchor = end if end > now else now
|
||
cal_days = max(1, (anchor.date() - start.date()).days + 1)
|
||
# 1영업일 ≈ 390분 + 여유. 주말 포함 캘린더일 보정
|
||
n = int(cal_days * 390 * 1.15) + 60
|
||
return min(cap, max(150, n))
|
||
|
||
|
||
def load_kiwoom_credentials(db: Any = None) -> Tuple[str, str, bool]:
|
||
"""시세(ka10080)용 키움 키 — 기본 **실키** (KIS_MOCK 매매와 분리).
|
||
|
||
``POST_SELL_CANDLE_FORCE_MOCK=true`` 일 때만 모의 키.
|
||
"""
|
||
from kis_trader.utils.env import get_env_bool as _geb
|
||
from kis_trader.utils.env import get_env_from_db
|
||
|
||
# 분봉 차트는 시세 → 실키 기본 (주문 모의와 무관)
|
||
force_mock = _geb("POST_SELL_CANDLE_FORCE_MOCK", False)
|
||
is_mock = bool(force_mock)
|
||
if is_mock:
|
||
key = str(get_env_from_db("KIWOOM_APP_KEY_MOCK", "") or "").strip()
|
||
sec = str(get_env_from_db("KIWOOM_APP_SECRET_MOCK", "") or "").strip()
|
||
else:
|
||
key = str(get_env_from_db("KIWOOM_APP_KEY_REAL", "") or "").strip()
|
||
sec = str(get_env_from_db("KIWOOM_APP_SECRET_REAL", "") or "").strip()
|
||
if not key or not sec:
|
||
key = str(get_env_from_db("KIWOOM_APP_KEY", "") or "").strip()
|
||
sec = str(get_env_from_db("KIWOOM_APP_SECRET", "") or "").strip()
|
||
if (not key or not sec) and db is not None:
|
||
try:
|
||
latest = db.get_latest_env()
|
||
snap = (latest or {}).get("snapshot") or {}
|
||
if is_mock:
|
||
key = str(snap.get("KIWOOM_APP_KEY_MOCK") or key).strip()
|
||
sec = str(snap.get("KIWOOM_APP_SECRET_MOCK") or sec).strip()
|
||
else:
|
||
key = str(snap.get("KIWOOM_APP_KEY_REAL") or key).strip()
|
||
sec = str(snap.get("KIWOOM_APP_SECRET_REAL") or sec).strip()
|
||
if not key or not sec:
|
||
key = str(snap.get("KIWOOM_APP_KEY") or key).strip()
|
||
sec = str(snap.get("KIWOOM_APP_SECRET") or sec).strip()
|
||
except Exception:
|
||
pass
|
||
return key, sec, bool(is_mock)
|
||
|
||
|
||
def _upsert_df_rows(db: Any, code: str, tf_min: int, rows: List[Dict[str, Any]]) -> int:
|
||
if not rows:
|
||
return 0
|
||
from kis_trader.ws.candle_series import normalize_source_channel
|
||
now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
payload = []
|
||
for rec in rows:
|
||
try:
|
||
ct = str(rec.get("candle_time") or rec.get("time") or "")[:12]
|
||
if len(ct) < 12:
|
||
continue
|
||
src, ch = normalize_source_channel(
|
||
str(rec.get("source") or "kiwoom"),
|
||
str(rec.get("channel") or "rest"),
|
||
)
|
||
payload.append((
|
||
code,
|
||
int(tf_min),
|
||
ct,
|
||
float(rec["open"]),
|
||
float(rec["high"]),
|
||
float(rec["low"]),
|
||
float(rec["close"]),
|
||
int(float(rec.get("volume") or 0)),
|
||
None, None, None,
|
||
1,
|
||
src,
|
||
ch,
|
||
now_str,
|
||
))
|
||
except Exception:
|
||
continue
|
||
if not payload:
|
||
return 0
|
||
sql = _insert_sql()
|
||
with db.conn._lock:
|
||
db.conn._ensure_connected()
|
||
cur = db.conn._conn.cursor()
|
||
cur.executemany(sql, payload)
|
||
db.conn._conn.commit()
|
||
return len(payload)
|
||
|
||
|
||
def count_confirmed_1m(db: Any, code: str, start_key: str, end_key: str) -> int:
|
||
row = db.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ws_candles "
|
||
"WHERE timeframe=1 AND code=%s AND candle_time >= %s AND candle_time <= %s "
|
||
"AND is_confirmed=1",
|
||
(code, start_key[:12], end_key[:12]),
|
||
).fetchone()
|
||
return int((row or {}).get("n") or 0)
|
||
|
||
|
||
def backfill_hold_window(
|
||
db: Any,
|
||
code: str,
|
||
start_raw: Any,
|
||
end_raw: Any,
|
||
*,
|
||
kiwoom_key: str = "",
|
||
kiwoom_secret: str = "",
|
||
is_mock: Optional[bool] = None,
|
||
rollup_3m: Optional[bool] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
``[start, end]`` 1분봉을 ka10080 으로 조회해 ws_candles UPSERT.
|
||
반환: before/after/upserted/rollup3m/ok/error
|
||
"""
|
||
code = str(code or "").strip()
|
||
start_key = _dt_to_candle_key(start_raw)
|
||
end_key = _dt_to_candle_key(end_raw)
|
||
out: Dict[str, Any] = {
|
||
"code": code,
|
||
"start": start_key,
|
||
"end": end_key,
|
||
"before": 0,
|
||
"after": 0,
|
||
"upserted": 0,
|
||
"rollup3m": 0,
|
||
"ok": False,
|
||
"error": "",
|
||
}
|
||
if not code or len(start_key) < 12 or len(end_key) < 12:
|
||
out["error"] = "bad_range"
|
||
return out
|
||
if end_key < start_key:
|
||
start_key, end_key = end_key, start_key
|
||
out["start"], out["end"] = start_key, end_key
|
||
|
||
before = count_confirmed_1m(db, code, start_key, end_key)
|
||
out["before"] = before
|
||
|
||
key, sec, mock = kiwoom_key, kiwoom_secret, is_mock
|
||
if not key or not sec or mock is None:
|
||
k2, s2, m2 = load_kiwoom_credentials(db)
|
||
key = key or k2
|
||
sec = sec or s2
|
||
mock = m2 if mock is None else mock
|
||
if not key or not sec:
|
||
out["error"] = "no_kiwoom_keys"
|
||
return out
|
||
|
||
n_req = _bars_needed(start_key, end_key)
|
||
try:
|
||
from kis_trader.ws.kis_ws import get_kiwoom_candles_df
|
||
|
||
df = get_kiwoom_candles_df(
|
||
code, 1, key, sec, is_mock=bool(mock), n=n_req,
|
||
)
|
||
except Exception as e:
|
||
out["error"] = f"ka10080:{e}"
|
||
logger.warning("📦 보유구간 백필 REST 실패 %s: %s", code, e)
|
||
return out
|
||
|
||
if df is None or getattr(df, "empty", True):
|
||
out["error"] = "empty_df"
|
||
out["after"] = before
|
||
return out
|
||
|
||
rows_1m: List[Dict[str, Any]] = []
|
||
for _, rec in df.iterrows():
|
||
ct = str(rec.get("time") or "")[:12]
|
||
if len(ct) < 12 or ct < start_key or ct > end_key:
|
||
continue
|
||
cl = float(rec.get("close") or 0)
|
||
if cl <= 0:
|
||
continue
|
||
rows_1m.append({
|
||
"candle_time": ct,
|
||
"open": float(rec.get("open") or cl),
|
||
"high": float(rec.get("high") or cl),
|
||
"low": float(rec.get("low") or cl),
|
||
"close": cl,
|
||
"volume": int(float(rec.get("volume") or 0)),
|
||
"source": "kiwoom",
|
||
"channel": "rest",
|
||
})
|
||
|
||
try:
|
||
upserted = _upsert_df_rows(db, code, 1, rows_1m)
|
||
except Exception as e:
|
||
out["error"] = f"upsert:{e}"
|
||
logger.warning("📦 보유구간 백필 UPSERT 실패 %s: %s", code, e)
|
||
return out
|
||
out["upserted"] = upserted
|
||
|
||
do_rollup = post_sell_candle_rollup_3m_enabled() if rollup_3m is None else bool(rollup_3m)
|
||
if do_rollup and rows_1m:
|
||
try:
|
||
from kis_trader.engine.candle_rollup import rollup_1m_bars_to_tf
|
||
|
||
bars3 = rollup_1m_bars_to_tf(rows_1m, 3)
|
||
for b in bars3:
|
||
b["source"] = "kiwoom"
|
||
b["channel"] = "rollup"
|
||
out["rollup3m"] = _upsert_df_rows(db, code, 3, bars3)
|
||
except Exception as e:
|
||
logger.debug("3M 롤업 스킵 %s: %s", code, e)
|
||
|
||
after = count_confirmed_1m(db, code, start_key, end_key)
|
||
out["after"] = after
|
||
out["ok"] = True
|
||
logger.info(
|
||
"📦 보유구간 백필 %s %s~%s | 1M %d→%d (upsert %d) 3M+%d",
|
||
code, start_key, end_key, before, after, upserted, int(out["rollup3m"]),
|
||
)
|
||
return out
|
||
|
||
|
||
def schedule_post_sell_backfill(
|
||
*,
|
||
code: str,
|
||
buy_date: Any,
|
||
sell_date: Any = None,
|
||
strategy: str = "",
|
||
) -> None:
|
||
"""매도 체결 후 비동기 1회 백필 (주문 스레드 비차단)."""
|
||
if not post_sell_candle_backfill_enabled():
|
||
return
|
||
code = str(code or "").strip()
|
||
if not code:
|
||
return
|
||
end = sell_date or datetime.now()
|
||
|
||
def _worker() -> None:
|
||
try:
|
||
# 서버 부하 완충 — 실매 체결 직후 REST 폭주 방지
|
||
lo = float(get_env_float("POST_SELL_CANDLE_SLEEP_MIN_SEC", 1.0))
|
||
hi = float(get_env_float("POST_SELL_CANDLE_SLEEP_MAX_SEC", 3.0))
|
||
if hi < lo:
|
||
hi = lo
|
||
time.sleep(random.uniform(lo, hi))
|
||
from database import TradeDB
|
||
|
||
db = TradeDB()
|
||
try:
|
||
backfill_hold_window(db, code, buy_date, end)
|
||
finally:
|
||
try:
|
||
db.close()
|
||
except Exception:
|
||
pass
|
||
except Exception as e:
|
||
logger.warning("📦 post-sell 백필 워커 예외 [%s/%s]: %s", strategy, code, e)
|
||
|
||
threading.Thread(
|
||
target=_worker,
|
||
name=f"post_sell_candle_{code}",
|
||
daemon=True,
|
||
).start()
|
||
|
||
|
||
def backfill_trades_from_db(
|
||
db: Any,
|
||
*,
|
||
buy_date_like: str = "2026-07-16%",
|
||
strategies: Optional[Sequence[str]] = None,
|
||
include_active: bool = True,
|
||
active_max_age_days: int = 5,
|
||
) -> List[Dict[str, Any]]:
|
||
"""
|
||
trade_history(청산) + 최근 active_trades(보유) 보유구간 즉시 백필.
|
||
``buy_date_like`` — pymysql 바인딩용 (예: '2026-07-16%').
|
||
"""
|
||
key, sec, mock = load_kiwoom_credentials(db)
|
||
results: List[Dict[str, Any]] = []
|
||
sleep_lo = float(get_env_float("POST_SELL_CANDLE_SLEEP_MIN_SEC", 1.0))
|
||
sleep_hi = float(get_env_float("POST_SELL_CANDLE_SLEEP_MAX_SEC", 3.0))
|
||
if sleep_hi < sleep_lo:
|
||
sleep_hi = sleep_lo
|
||
|
||
sql = (
|
||
"SELECT code, name, strategy, buy_date, sell_date FROM trade_history "
|
||
"WHERE buy_date LIKE %s"
|
||
)
|
||
params: List[Any] = [buy_date_like]
|
||
if strategies:
|
||
ph = ",".join(["%s"] * len(strategies))
|
||
sql += f" AND strategy IN ({ph})"
|
||
params.extend(list(strategies))
|
||
sql += " ORDER BY buy_date"
|
||
closed = db.conn.execute(sql, tuple(params)).fetchall()
|
||
|
||
jobs: List[Tuple[str, Any, Any, str]] = []
|
||
for r in closed:
|
||
d = dict(r)
|
||
jobs.append((
|
||
str(d.get("code") or ""),
|
||
d.get("buy_date"),
|
||
d.get("sell_date"),
|
||
str(d.get("strategy") or ""),
|
||
))
|
||
|
||
if include_active:
|
||
act_sql = (
|
||
"SELECT code, name, strategy, buy_date FROM active_trades "
|
||
"WHERE buy_date >= DATE_SUB(NOW(), INTERVAL %s DAY)"
|
||
)
|
||
act_params: List[Any] = [int(active_max_age_days)]
|
||
if strategies:
|
||
ph = ",".join(["%s"] * len(strategies))
|
||
act_sql += f" AND strategy IN ({ph})"
|
||
act_params.extend(list(strategies))
|
||
for r in db.conn.execute(act_sql, tuple(act_params)).fetchall():
|
||
d = dict(r)
|
||
jobs.append((
|
||
str(d.get("code") or ""),
|
||
d.get("buy_date"),
|
||
datetime.now(),
|
||
str(d.get("strategy") or ""),
|
||
))
|
||
|
||
# 동일 code+구간 중복 제거 (같은 날 재진입은 구간 합치지 않고 각각)
|
||
seen = set()
|
||
uniq_jobs = []
|
||
for code, b, e, sid in jobs:
|
||
if not code:
|
||
continue
|
||
sk = (_dt_to_candle_key(b), _dt_to_candle_key(e), code, sid)
|
||
if sk in seen:
|
||
continue
|
||
seen.add(sk)
|
||
uniq_jobs.append((code, b, e, sid))
|
||
|
||
logger.info(
|
||
"📦 즉시 백필 시작: %d건 (closed=%d active포함=%s)",
|
||
len(uniq_jobs), len(closed), include_active,
|
||
)
|
||
for i, (code, b, e, sid) in enumerate(uniq_jobs, 1):
|
||
if i > 1:
|
||
time.sleep(random.uniform(sleep_lo, sleep_hi))
|
||
st = backfill_hold_window(
|
||
db, code, b, e,
|
||
kiwoom_key=key, kiwoom_secret=sec, is_mock=mock,
|
||
)
|
||
st["strategy"] = sid
|
||
results.append(st)
|
||
return results
|