253c95e2c2
- 백테스트 및 실거래 시 틱과 호가의 벤더 출처(ob_source, entry_source) 기록 및 추적 강화 (tail_engine.py) - 웹 UI '체결디버그'에 [틱:kis / 호가:ls] 형태로 데이터 출처를 직관적으로 표출 (backtest.js, backtest.html) - LS WebSocket 구독 100건 제한 하드코딩 해제 및 env_config_ext 연동 (ls_ws.py) - 기타 백테스트 웹 및 DB 관련 최적화 적용
453 lines
15 KiB
Python
453 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
전략 백테 CLI 웹 잡 (꼬리·스캘핑·돌파·모멘텀 공용).
|
|
|
|
- subprocess + start_new_session + reaper(wait) → 좀비(Z) 방지
|
|
- 전역 1잡만 running (전략 간 충돌·API 폭주 방지)
|
|
- progress.json + job meta 디스크 저장 → 탭 이동·새로고침에도 폴링 가능
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
JOBS_DIR = ROOT / "logs" / "bt_web_jobs"
|
|
RESULTS_DIR = ROOT / "kis_trader" / "backtest" / "results"
|
|
PY = ROOT / ".venv" / "bin" / "python"
|
|
CLI = ROOT / "scripts" / "run_strategy_backtest_cli.py"
|
|
TAIL_CLI = ROOT / "scripts" / "run_tail_backtest_cli.py"
|
|
|
|
STRATEGIES = ("tail", "scalp", "breakout", "momentum")
|
|
LABELS = {
|
|
"tail": "꼬리 백테",
|
|
"scalp": "스캘핑 백테",
|
|
"breakout": "돌파 백테",
|
|
"momentum": "모멘텀 백테",
|
|
}
|
|
|
|
|
|
def _now_iso() -> str:
|
|
return datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
|
|
|
|
|
def _ensure_dirs() -> None:
|
|
JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
|
(ROOT / "logs").mkdir(parents=True, exist_ok=True)
|
|
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def _job_path(job_id: str) -> Path:
|
|
return JOBS_DIR / f"{job_id}.json"
|
|
|
|
|
|
def _progress_path(job_id: str) -> Path:
|
|
return JOBS_DIR / f"{job_id}.progress.json"
|
|
|
|
|
|
def save_job(meta: Dict[str, Any]) -> None:
|
|
_ensure_dirs()
|
|
jid = str(meta.get("job_id") or "")
|
|
if not jid:
|
|
return
|
|
p = _job_path(jid)
|
|
tmp = p.with_suffix(".tmp")
|
|
tmp.write_text(json.dumps(meta, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
|
tmp.replace(p)
|
|
|
|
|
|
def _legacy_job_path(job_id: str) -> Path:
|
|
return ROOT / "logs" / "tail_bt_web_jobs" / f"{job_id}.json"
|
|
|
|
|
|
def load_job(job_id: str) -> Optional[Dict[str, Any]]:
|
|
for p in (_job_path(job_id), _legacy_job_path(job_id)):
|
|
if not p.is_file():
|
|
continue
|
|
try:
|
|
return json.loads(p.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
continue
|
|
return None
|
|
|
|
|
|
def _try_reap_child(pid: int) -> None:
|
|
try:
|
|
os.waitpid(int(pid), os.WNOHANG)
|
|
except (ChildProcessError, OSError, ValueError):
|
|
pass
|
|
|
|
|
|
def _pid_alive(pid: int) -> bool:
|
|
"""살아 있으면 True. 좀비(Z)는 wait 후 False."""
|
|
if pid <= 0:
|
|
return False
|
|
try:
|
|
os.kill(pid, 0)
|
|
except OSError:
|
|
return False
|
|
try:
|
|
raw = Path("/proc/%d/stat" % int(pid)).read_text(encoding="utf-8", errors="replace")
|
|
rp = raw.rfind(")")
|
|
if rp >= 0 and rp + 2 < len(raw):
|
|
state = raw[rp + 2 : rp + 3]
|
|
if state == "Z":
|
|
_try_reap_child(int(pid))
|
|
return False
|
|
except Exception:
|
|
pass
|
|
return True
|
|
|
|
|
|
def read_progress(job_id: str) -> Dict[str, Any]:
|
|
for p in (
|
|
_progress_path(job_id),
|
|
ROOT / "logs" / "tail_bt_web_jobs" / f"{job_id}.progress.json",
|
|
):
|
|
if not p.is_file():
|
|
continue
|
|
try:
|
|
return json.loads(p.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
continue
|
|
return {"pct": 0, "phase": "init", "message": ""}
|
|
|
|
|
|
def refresh_job(job_id: str) -> Optional[Dict[str, Any]]:
|
|
m = load_job(job_id)
|
|
if not m:
|
|
return None
|
|
pid = int(m.get("pid") or 0)
|
|
prog = read_progress(job_id)
|
|
m["progress"] = prog
|
|
m["pid_alive"] = _pid_alive(pid)
|
|
st = str(m.get("status") or "")
|
|
if st == "running":
|
|
if not m["pid_alive"]:
|
|
rj = m.get("result_json")
|
|
if rj and Path(str(rj)).is_file():
|
|
m["status"] = "done"
|
|
m["finished_at"] = m.get("finished_at") or _now_iso()
|
|
try:
|
|
data = json.loads(Path(str(rj)).read_text(encoding="utf-8"))
|
|
m["result_summary"] = data.get("summary") or {}
|
|
m["progress"] = {
|
|
"pct": 100,
|
|
"phase": "done",
|
|
"message": "완료",
|
|
"step": prog.get("step"),
|
|
"total": prog.get("total"),
|
|
}
|
|
except Exception:
|
|
m["progress"] = {"pct": 100, "phase": "done", "message": "완료"}
|
|
else:
|
|
err = ""
|
|
logp = m.get("log_path")
|
|
if logp and Path(str(logp)).is_file():
|
|
try:
|
|
tail = Path(str(logp)).read_text(encoding="utf-8", errors="replace")[-2000:]
|
|
if "Traceback" in tail or "실패" in tail:
|
|
err = tail.strip().splitlines()[-1][:200]
|
|
except Exception:
|
|
pass
|
|
m["status"] = "error"
|
|
m["error"] = err or "프로세스 종료·결과 없음"
|
|
m["finished_at"] = _now_iso()
|
|
m["progress"] = {
|
|
"pct": int(prog.get("pct") or 0),
|
|
"phase": "error",
|
|
"message": m["error"],
|
|
}
|
|
save_job(m)
|
|
return m
|
|
|
|
|
|
def list_jobs(limit: int = 30) -> List[Dict[str, Any]]:
|
|
_ensure_dirs()
|
|
# 구버전 tail_bt_web_jobs 도 함께 보여 줌
|
|
dirs = [JOBS_DIR, ROOT / "logs" / "tail_bt_web_jobs"]
|
|
seen = set()
|
|
rows: List[Dict[str, Any]] = []
|
|
paths: List[Path] = []
|
|
for d in dirs:
|
|
if d.is_dir():
|
|
paths.extend(d.glob("*.json"))
|
|
for p in sorted(paths, key=lambda x: x.stat().st_mtime, reverse=True):
|
|
if p.name.endswith(".progress.json") or p.name.endswith(".tmp") or ".progress." in p.name:
|
|
continue
|
|
try:
|
|
m = json.loads(p.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
continue
|
|
if not isinstance(m, dict) or not m.get("job_id"):
|
|
continue
|
|
jid = str(m["job_id"])
|
|
if jid in seen:
|
|
continue
|
|
seen.add(jid)
|
|
# 새 디렉터리 우선 refresh
|
|
if (_job_path(jid)).is_file():
|
|
rows.append(refresh_job(jid) or m)
|
|
else:
|
|
# legacy tail path — 최소 필드만
|
|
m["progress"] = m.get("progress") or {}
|
|
m["pid_alive"] = _pid_alive(int(m.get("pid") or 0))
|
|
rows.append(m)
|
|
if len(rows) >= limit:
|
|
break
|
|
return rows
|
|
|
|
|
|
def find_running() -> List[Dict[str, Any]]:
|
|
out = []
|
|
for m in list_jobs(limit=80):
|
|
if m.get("status") == "running" and m.get("pid_alive"):
|
|
out.append(m)
|
|
return out
|
|
|
|
|
|
def _spawn_reaper(proc: subprocess.Popen, job_id: str, log_f: Any) -> None:
|
|
def _run() -> None:
|
|
rc: Optional[int] = None
|
|
try:
|
|
rc = int(proc.wait())
|
|
except Exception:
|
|
try:
|
|
rc = int(proc.poll()) if proc.poll() is not None else None
|
|
except Exception:
|
|
rc = None
|
|
try:
|
|
if log_f is not None and hasattr(log_f, "closed") and not log_f.closed:
|
|
log_f.flush()
|
|
log_f.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
meta = load_job(job_id)
|
|
if not meta:
|
|
return
|
|
meta["exit_code"] = rc
|
|
if rc not in (None, 0) and meta.get("status") == "running":
|
|
# result 있으면 done 우선
|
|
rj = meta.get("result_json")
|
|
if not (rj and Path(str(rj)).is_file()):
|
|
meta["error"] = meta.get("error") or f"process exit_code={rc}"
|
|
save_job(meta)
|
|
refresh_job(job_id)
|
|
except Exception:
|
|
pass
|
|
|
|
threading.Thread(target=_run, name=f"bt-reap-{job_id}", daemon=True).start()
|
|
|
|
|
|
def start_bt_job(
|
|
*,
|
|
strategy: str,
|
|
start: str,
|
|
end: str,
|
|
timeframe: Optional[int] = None,
|
|
universe: str = "history",
|
|
universe_history_source: Optional[str] = None,
|
|
tick_db: Optional[bool] = None,
|
|
tick_exit: Optional[bool] = None,
|
|
orderbook_filter: str = "auto",
|
|
params_json: Optional[str] = None,
|
|
params_override: Optional[Dict[str, Any]] = None,
|
|
env_timeline: bool = False,
|
|
) -> Dict[str, Any]:
|
|
_ensure_dirs()
|
|
strat = str(strategy or "").strip().lower()
|
|
if strat not in STRATEGIES:
|
|
raise ValueError(f"strategy 는 {STRATEGIES} 중 하나")
|
|
|
|
running = find_running()
|
|
if running:
|
|
r0 = running[0]
|
|
raise RuntimeError(
|
|
f"이미 실행 중: {r0.get('label') or r0.get('strategy')} job={r0.get('job_id')}. "
|
|
"끝난 뒤 다시 시작하거나 중지하세요."
|
|
)
|
|
|
|
if not PY.is_file():
|
|
raise RuntimeError(f"python 없음: {PY}")
|
|
if not CLI.is_file() and not TAIL_CLI.is_file():
|
|
raise RuntimeError(f"CLI 없음: {CLI}")
|
|
|
|
from kis_trader.utils.kr_trading_day import clamp_to_prev_kr_trading_day
|
|
|
|
start = clamp_to_prev_kr_trading_day(str(start or "").strip())
|
|
end = clamp_to_prev_kr_trading_day(str(end or "").strip())
|
|
if start > end:
|
|
start, end = end, start
|
|
if not start or not end:
|
|
raise ValueError("start/end 필요")
|
|
|
|
univ = str(universe or "history").strip().lower()
|
|
if univ not in ("history", "all", "sim"):
|
|
univ = "history"
|
|
hist_src = str(universe_history_source or "").strip().lower()
|
|
if hist_src not in ("", "kiwoom", "ls"):
|
|
hist_src = ""
|
|
|
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
job_id = f"{strat}_bt_{start.replace('-', '')}_{end.replace('-', '')}_{ts}"
|
|
log_path = ROOT / "logs" / f"{job_id}.log"
|
|
result_json = RESULTS_DIR / f"{job_id}.json"
|
|
progress_file = _progress_path(job_id)
|
|
|
|
params_json_path = str(params_json or "").strip() or None
|
|
if params_override and isinstance(params_override, dict) and params_override:
|
|
pj_dir = ROOT / "logs" / "bt_job_params"
|
|
pj_dir.mkdir(parents=True, exist_ok=True)
|
|
pj = pj_dir / f"{job_id}_params.json"
|
|
pj.write_text(json.dumps(params_override, ensure_ascii=False), encoding="utf-8")
|
|
params_json_path = str(pj)
|
|
|
|
# tail 은 세밀 progress 있는 전용 CLI 우선
|
|
if strat == "tail" and TAIL_CLI.is_file():
|
|
cmd = [
|
|
str(PY), "-u", str(TAIL_CLI),
|
|
"--start", start,
|
|
"--end", end,
|
|
"--timeframe", str(int(timeframe or 3)),
|
|
"--universe", "history" if univ == "sim" else univ,
|
|
"--job-id", job_id,
|
|
"--out-dir", str(RESULTS_DIR),
|
|
"--progress-file", str(progress_file),
|
|
"--orderbook-filter", str(orderbook_filter or "auto"),
|
|
]
|
|
if tick_db is not None:
|
|
cmd.extend(["--tick-db", "1" if tick_db else "0"])
|
|
if tick_exit is not None:
|
|
cmd.extend(["--tick-exit", "1" if tick_exit else "0"])
|
|
if params_json_path:
|
|
cmd.extend(["--params-json", str(params_json_path)])
|
|
if env_timeline:
|
|
cmd.append("--env-timeline")
|
|
if hist_src:
|
|
cmd.extend(["--universe-history-source", hist_src])
|
|
else:
|
|
if not CLI.is_file():
|
|
raise RuntimeError(f"CLI 없음: {CLI}")
|
|
cmd = [
|
|
str(PY), "-u", str(CLI),
|
|
"--strategy", strat,
|
|
"--start", start,
|
|
"--end", end,
|
|
"--universe", univ,
|
|
"--job-id", job_id,
|
|
"--out-dir", str(RESULTS_DIR),
|
|
"--progress-file", str(progress_file),
|
|
"--orderbook-filter", str(orderbook_filter or "auto"),
|
|
]
|
|
if timeframe is not None:
|
|
cmd.extend(["--timeframe", str(int(timeframe))])
|
|
if tick_db is not None:
|
|
cmd.extend(["--tick-db", "1" if tick_db else "0"])
|
|
if params_json_path:
|
|
cmd.extend(["--params-json", str(params_json_path)])
|
|
if env_timeline:
|
|
cmd.append("--env-timeline")
|
|
if hist_src:
|
|
cmd.extend(["--universe-history-source", hist_src])
|
|
|
|
progress_file.write_text(
|
|
json.dumps({"pct": 1, "phase": "starting", "message": "시작"}, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
log_f = open(log_path, "w", encoding="utf-8")
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
cwd=str(ROOT),
|
|
stdout=log_f,
|
|
stderr=subprocess.STDOUT,
|
|
start_new_session=True,
|
|
)
|
|
meta = {
|
|
"job_id": job_id,
|
|
"kind": "strategy_bt_cli",
|
|
"label": LABELS.get(strat, strat),
|
|
"strategy": strat,
|
|
"start": start,
|
|
"end": end,
|
|
"timeframe": int(timeframe) if timeframe is not None else None,
|
|
"universe": univ,
|
|
"universe_history_source": hist_src or "kiwoom",
|
|
"status": "running",
|
|
"pid": proc.pid,
|
|
"log_path": str(log_path),
|
|
"result_json": str(result_json),
|
|
"progress_file": str(progress_file),
|
|
"started_at": _now_iso(),
|
|
"started_ts": time.time(),
|
|
"cmd": " ".join(cmd),
|
|
"progress": {"pct": 1, "phase": "starting", "message": "시작"},
|
|
}
|
|
save_job(meta)
|
|
_spawn_reaper(proc, job_id, log_f)
|
|
return meta
|
|
|
|
|
|
def stop_bt_job(job_id: str) -> Dict[str, Any]:
|
|
m = load_job(job_id)
|
|
if not m:
|
|
# legacy tail dir
|
|
legacy = ROOT / "logs" / "tail_bt_web_jobs" / f"{job_id}.json"
|
|
if legacy.is_file():
|
|
try:
|
|
m = json.loads(legacy.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
m = None
|
|
if not m:
|
|
raise RuntimeError("job 없음")
|
|
pid = int(m.get("pid") or 0)
|
|
if pid > 0 and _pid_alive(pid):
|
|
try:
|
|
os.killpg(pid, signal.SIGTERM)
|
|
except Exception:
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
except Exception:
|
|
pass
|
|
# 짧게 대기 후 강제
|
|
time.sleep(0.4)
|
|
if _pid_alive(pid):
|
|
try:
|
|
os.killpg(pid, signal.SIGKILL)
|
|
except Exception:
|
|
try:
|
|
os.kill(pid, signal.SIGKILL)
|
|
except Exception:
|
|
pass
|
|
_try_reap_child(pid)
|
|
m["status"] = "error"
|
|
m["error"] = "사용자 중지"
|
|
m["finished_at"] = _now_iso()
|
|
m["progress"] = {
|
|
"pct": int((m.get("progress") or {}).get("pct") or 0),
|
|
"phase": "stopped",
|
|
"message": "중지",
|
|
}
|
|
# 새 경로에 저장 (legacy 잡도 새 폴더로 옮김)
|
|
save_job(m)
|
|
return m
|
|
|
|
|
|
# ── 하위호환 별칭 (tail_bt_web_jobs) ─────────────────────────────────────────
|
|
def start_tail_bt_job(**kwargs: Any) -> Dict[str, Any]:
|
|
kwargs = dict(kwargs)
|
|
kwargs["strategy"] = "tail"
|
|
return start_bt_job(**kwargs)
|
|
|
|
|
|
def stop_tail_bt_job(job_id: str) -> Dict[str, Any]:
|
|
return stop_bt_job(job_id)
|