ls증권 히스토리 구독 넣음
This commit is contained in:
Your Name
443
kis_trader/backtest/bt_web_jobs.py
Normal file
443
kis_trader/backtest/bt_web_jobs.py
Normal file
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
전략 백테 CLI 웹 잡 (꼬리·스캘핑·돌파·모멘텀 공용).
|
||||
|
||||
- subprocess + start_new_session + reaper(wait) → 좀비(Z) 방지
|
||||
- 전역 1잡만 running (전략 간 충돌·API 폭주 방지)
|
||||
- progress.json + job meta 디스크 저장 → 탭 이동·새로고침에도 폴링 가능
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
JOBS_DIR = ROOT / "logs" / "bt_web_jobs"
|
||||
RESULTS_DIR = ROOT / "kis_trader" / "backtest" / "results"
|
||||
PY = ROOT / ".venv" / "bin" / "python"
|
||||
CLI = ROOT / "scripts" / "run_strategy_backtest_cli.py"
|
||||
TAIL_CLI = ROOT / "scripts" / "run_tail_backtest_cli.py"
|
||||
|
||||
STRATEGIES = ("tail", "scalp", "breakout", "momentum")
|
||||
LABELS = {
|
||||
"tail": "꼬리 백테",
|
||||
"scalp": "스캘핑 백테",
|
||||
"breakout": "돌파 백테",
|
||||
"momentum": "모멘텀 백테",
|
||||
}
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
|
||||
def _ensure_dirs() -> None:
|
||||
JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(ROOT / "logs").mkdir(parents=True, exist_ok=True)
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _job_path(job_id: str) -> Path:
|
||||
return JOBS_DIR / f"{job_id}.json"
|
||||
|
||||
|
||||
def _progress_path(job_id: str) -> Path:
|
||||
return JOBS_DIR / f"{job_id}.progress.json"
|
||||
|
||||
|
||||
def save_job(meta: Dict[str, Any]) -> None:
|
||||
_ensure_dirs()
|
||||
jid = str(meta.get("job_id") or "")
|
||||
if not jid:
|
||||
return
|
||||
p = _job_path(jid)
|
||||
tmp = p.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(meta, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(p)
|
||||
|
||||
|
||||
def _legacy_job_path(job_id: str) -> Path:
|
||||
return ROOT / "logs" / "tail_bt_web_jobs" / f"{job_id}.json"
|
||||
|
||||
|
||||
def load_job(job_id: str) -> Optional[Dict[str, Any]]:
|
||||
for p in (_job_path(job_id), _legacy_job_path(job_id)):
|
||||
if not p.is_file():
|
||||
continue
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _try_reap_child(pid: int) -> None:
|
||||
try:
|
||||
os.waitpid(int(pid), os.WNOHANG)
|
||||
except (ChildProcessError, OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _pid_alive(pid: int) -> bool:
|
||||
"""살아 있으면 True. 좀비(Z)는 wait 후 False."""
|
||||
if pid <= 0:
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
raw = Path("/proc/%d/stat" % int(pid)).read_text(encoding="utf-8", errors="replace")
|
||||
rp = raw.rfind(")")
|
||||
if rp >= 0 and rp + 2 < len(raw):
|
||||
state = raw[rp + 2 : rp + 3]
|
||||
if state == "Z":
|
||||
_try_reap_child(int(pid))
|
||||
return False
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def read_progress(job_id: str) -> Dict[str, Any]:
|
||||
for p in (
|
||||
_progress_path(job_id),
|
||||
ROOT / "logs" / "tail_bt_web_jobs" / f"{job_id}.progress.json",
|
||||
):
|
||||
if not p.is_file():
|
||||
continue
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
return {"pct": 0, "phase": "init", "message": ""}
|
||||
|
||||
|
||||
def refresh_job(job_id: str) -> Optional[Dict[str, Any]]:
|
||||
m = load_job(job_id)
|
||||
if not m:
|
||||
return None
|
||||
pid = int(m.get("pid") or 0)
|
||||
prog = read_progress(job_id)
|
||||
m["progress"] = prog
|
||||
m["pid_alive"] = _pid_alive(pid)
|
||||
st = str(m.get("status") or "")
|
||||
if st == "running":
|
||||
if not m["pid_alive"]:
|
||||
rj = m.get("result_json")
|
||||
if rj and Path(str(rj)).is_file():
|
||||
m["status"] = "done"
|
||||
m["finished_at"] = m.get("finished_at") or _now_iso()
|
||||
try:
|
||||
data = json.loads(Path(str(rj)).read_text(encoding="utf-8"))
|
||||
m["result_summary"] = data.get("summary") or {}
|
||||
m["progress"] = {
|
||||
"pct": 100,
|
||||
"phase": "done",
|
||||
"message": "완료",
|
||||
"step": prog.get("step"),
|
||||
"total": prog.get("total"),
|
||||
}
|
||||
except Exception:
|
||||
m["progress"] = {"pct": 100, "phase": "done", "message": "완료"}
|
||||
else:
|
||||
err = ""
|
||||
logp = m.get("log_path")
|
||||
if logp and Path(str(logp)).is_file():
|
||||
try:
|
||||
tail = Path(str(logp)).read_text(encoding="utf-8", errors="replace")[-2000:]
|
||||
if "Traceback" in tail or "실패" in tail:
|
||||
err = tail.strip().splitlines()[-1][:200]
|
||||
except Exception:
|
||||
pass
|
||||
m["status"] = "error"
|
||||
m["error"] = err or "프로세스 종료·결과 없음"
|
||||
m["finished_at"] = _now_iso()
|
||||
m["progress"] = {
|
||||
"pct": int(prog.get("pct") or 0),
|
||||
"phase": "error",
|
||||
"message": m["error"],
|
||||
}
|
||||
save_job(m)
|
||||
return m
|
||||
|
||||
|
||||
def list_jobs(limit: int = 30) -> List[Dict[str, Any]]:
|
||||
_ensure_dirs()
|
||||
# 구버전 tail_bt_web_jobs 도 함께 보여 줌
|
||||
dirs = [JOBS_DIR, ROOT / "logs" / "tail_bt_web_jobs"]
|
||||
seen = set()
|
||||
rows: List[Dict[str, Any]] = []
|
||||
paths: List[Path] = []
|
||||
for d in dirs:
|
||||
if d.is_dir():
|
||||
paths.extend(d.glob("*.json"))
|
||||
for p in sorted(paths, key=lambda x: x.stat().st_mtime, reverse=True):
|
||||
if p.name.endswith(".progress.json") or p.name.endswith(".tmp") or ".progress." in p.name:
|
||||
continue
|
||||
try:
|
||||
m = json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(m, dict) or not m.get("job_id"):
|
||||
continue
|
||||
jid = str(m["job_id"])
|
||||
if jid in seen:
|
||||
continue
|
||||
seen.add(jid)
|
||||
# 새 디렉터리 우선 refresh
|
||||
if (_job_path(jid)).is_file():
|
||||
rows.append(refresh_job(jid) or m)
|
||||
else:
|
||||
# legacy tail path — 최소 필드만
|
||||
m["progress"] = m.get("progress") or {}
|
||||
m["pid_alive"] = _pid_alive(int(m.get("pid") or 0))
|
||||
rows.append(m)
|
||||
if len(rows) >= limit:
|
||||
break
|
||||
return rows
|
||||
|
||||
|
||||
def find_running() -> List[Dict[str, Any]]:
|
||||
out = []
|
||||
for m in list_jobs(limit=80):
|
||||
if m.get("status") == "running" and m.get("pid_alive"):
|
||||
out.append(m)
|
||||
return out
|
||||
|
||||
|
||||
def _spawn_reaper(proc: subprocess.Popen, job_id: str, log_f: Any) -> None:
|
||||
def _run() -> None:
|
||||
rc: Optional[int] = None
|
||||
try:
|
||||
rc = int(proc.wait())
|
||||
except Exception:
|
||||
try:
|
||||
rc = int(proc.poll()) if proc.poll() is not None else None
|
||||
except Exception:
|
||||
rc = None
|
||||
try:
|
||||
if log_f is not None and hasattr(log_f, "closed") and not log_f.closed:
|
||||
log_f.flush()
|
||||
log_f.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
meta = load_job(job_id)
|
||||
if not meta:
|
||||
return
|
||||
meta["exit_code"] = rc
|
||||
if rc not in (None, 0) and meta.get("status") == "running":
|
||||
# result 있으면 done 우선
|
||||
rj = meta.get("result_json")
|
||||
if not (rj and Path(str(rj)).is_file()):
|
||||
meta["error"] = meta.get("error") or f"process exit_code={rc}"
|
||||
save_job(meta)
|
||||
refresh_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
threading.Thread(target=_run, name=f"bt-reap-{job_id}", daemon=True).start()
|
||||
|
||||
|
||||
def start_bt_job(
|
||||
*,
|
||||
strategy: str,
|
||||
start: str,
|
||||
end: str,
|
||||
timeframe: Optional[int] = None,
|
||||
universe: str = "history",
|
||||
universe_history_source: Optional[str] = None,
|
||||
tick_db: Optional[bool] = None,
|
||||
tick_exit: Optional[bool] = None,
|
||||
orderbook_filter: str = "off",
|
||||
params_json: Optional[str] = None,
|
||||
env_timeline: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
_ensure_dirs()
|
||||
strat = str(strategy or "").strip().lower()
|
||||
if strat not in STRATEGIES:
|
||||
raise ValueError(f"strategy 는 {STRATEGIES} 중 하나")
|
||||
|
||||
running = find_running()
|
||||
if running:
|
||||
r0 = running[0]
|
||||
raise RuntimeError(
|
||||
f"이미 실행 중: {r0.get('label') or r0.get('strategy')} job={r0.get('job_id')}. "
|
||||
"끝난 뒤 다시 시작하거나 중지하세요."
|
||||
)
|
||||
|
||||
if not PY.is_file():
|
||||
raise RuntimeError(f"python 없음: {PY}")
|
||||
if not CLI.is_file() and not TAIL_CLI.is_file():
|
||||
raise RuntimeError(f"CLI 없음: {CLI}")
|
||||
|
||||
from kis_trader.utils.kr_trading_day import clamp_to_prev_kr_trading_day
|
||||
|
||||
start = clamp_to_prev_kr_trading_day(str(start or "").strip())
|
||||
end = clamp_to_prev_kr_trading_day(str(end or "").strip())
|
||||
if start > end:
|
||||
start, end = end, start
|
||||
if not start or not end:
|
||||
raise ValueError("start/end 필요")
|
||||
|
||||
univ = str(universe or "history").strip().lower()
|
||||
if univ not in ("history", "all", "sim"):
|
||||
univ = "history"
|
||||
hist_src = str(universe_history_source or "").strip().lower()
|
||||
if hist_src not in ("", "kiwoom", "ls"):
|
||||
hist_src = ""
|
||||
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
job_id = f"{strat}_bt_{start.replace('-', '')}_{end.replace('-', '')}_{ts}"
|
||||
log_path = ROOT / "logs" / f"{job_id}.log"
|
||||
result_json = RESULTS_DIR / f"{job_id}.json"
|
||||
progress_file = _progress_path(job_id)
|
||||
|
||||
# tail 은 세밀 progress 있는 전용 CLI 우선
|
||||
if strat == "tail" and TAIL_CLI.is_file():
|
||||
cmd = [
|
||||
str(PY), "-u", str(TAIL_CLI),
|
||||
"--start", start,
|
||||
"--end", end,
|
||||
"--timeframe", str(int(timeframe or 3)),
|
||||
"--universe", "history" if univ == "sim" else univ,
|
||||
"--job-id", job_id,
|
||||
"--out-dir", str(RESULTS_DIR),
|
||||
"--progress-file", str(progress_file),
|
||||
"--orderbook-filter", str(orderbook_filter or "off"),
|
||||
]
|
||||
if tick_db is not None:
|
||||
cmd.extend(["--tick-db", "1" if tick_db else "0"])
|
||||
if tick_exit is not None:
|
||||
cmd.extend(["--tick-exit", "1" if tick_exit else "0"])
|
||||
if params_json:
|
||||
cmd.extend(["--params-json", str(params_json)])
|
||||
if env_timeline:
|
||||
cmd.append("--env-timeline")
|
||||
if hist_src:
|
||||
cmd.extend(["--universe-history-source", hist_src])
|
||||
else:
|
||||
if not CLI.is_file():
|
||||
raise RuntimeError(f"CLI 없음: {CLI}")
|
||||
cmd = [
|
||||
str(PY), "-u", str(CLI),
|
||||
"--strategy", strat,
|
||||
"--start", start,
|
||||
"--end", end,
|
||||
"--universe", univ,
|
||||
"--job-id", job_id,
|
||||
"--out-dir", str(RESULTS_DIR),
|
||||
"--progress-file", str(progress_file),
|
||||
"--orderbook-filter", str(orderbook_filter or "off"),
|
||||
]
|
||||
if timeframe is not None:
|
||||
cmd.extend(["--timeframe", str(int(timeframe))])
|
||||
if tick_db is not None:
|
||||
cmd.extend(["--tick-db", "1" if tick_db else "0"])
|
||||
if params_json:
|
||||
cmd.extend(["--params-json", str(params_json)])
|
||||
if env_timeline:
|
||||
cmd.append("--env-timeline")
|
||||
if hist_src:
|
||||
cmd.extend(["--universe-history-source", hist_src])
|
||||
|
||||
progress_file.write_text(
|
||||
json.dumps({"pct": 1, "phase": "starting", "message": "시작"}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
log_f = open(log_path, "w", encoding="utf-8")
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(ROOT),
|
||||
stdout=log_f,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
)
|
||||
meta = {
|
||||
"job_id": job_id,
|
||||
"kind": "strategy_bt_cli",
|
||||
"label": LABELS.get(strat, strat),
|
||||
"strategy": strat,
|
||||
"start": start,
|
||||
"end": end,
|
||||
"timeframe": int(timeframe) if timeframe is not None else None,
|
||||
"universe": univ,
|
||||
"universe_history_source": hist_src or "kiwoom",
|
||||
"status": "running",
|
||||
"pid": proc.pid,
|
||||
"log_path": str(log_path),
|
||||
"result_json": str(result_json),
|
||||
"progress_file": str(progress_file),
|
||||
"started_at": _now_iso(),
|
||||
"started_ts": time.time(),
|
||||
"cmd": " ".join(cmd),
|
||||
"progress": {"pct": 1, "phase": "starting", "message": "시작"},
|
||||
}
|
||||
save_job(meta)
|
||||
_spawn_reaper(proc, job_id, log_f)
|
||||
return meta
|
||||
|
||||
|
||||
def stop_bt_job(job_id: str) -> Dict[str, Any]:
|
||||
m = load_job(job_id)
|
||||
if not m:
|
||||
# legacy tail dir
|
||||
legacy = ROOT / "logs" / "tail_bt_web_jobs" / f"{job_id}.json"
|
||||
if legacy.is_file():
|
||||
try:
|
||||
m = json.loads(legacy.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
m = None
|
||||
if not m:
|
||||
raise RuntimeError("job 없음")
|
||||
pid = int(m.get("pid") or 0)
|
||||
if pid > 0 and _pid_alive(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGTERM)
|
||||
except Exception:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except Exception:
|
||||
pass
|
||||
# 짧게 대기 후 강제
|
||||
time.sleep(0.4)
|
||||
if _pid_alive(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except Exception:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except Exception:
|
||||
pass
|
||||
_try_reap_child(pid)
|
||||
m["status"] = "error"
|
||||
m["error"] = "사용자 중지"
|
||||
m["finished_at"] = _now_iso()
|
||||
m["progress"] = {
|
||||
"pct": int((m.get("progress") or {}).get("pct") or 0),
|
||||
"phase": "stopped",
|
||||
"message": "중지",
|
||||
}
|
||||
# 새 경로에 저장 (legacy 잡도 새 폴더로 옮김)
|
||||
save_job(m)
|
||||
return m
|
||||
|
||||
|
||||
# ── 하위호환 별칭 (tail_bt_web_jobs) ─────────────────────────────────────────
|
||||
def start_tail_bt_job(**kwargs: Any) -> Dict[str, Any]:
|
||||
kwargs = dict(kwargs)
|
||||
kwargs["strategy"] = "tail"
|
||||
return start_bt_job(**kwargs)
|
||||
|
||||
|
||||
def stop_tail_bt_job(job_id: str) -> Dict[str, Any]:
|
||||
return stop_bt_job(job_id)
|
||||
Reference in New Issue
Block a user