cb7e5037a0
Changes: - Introduced the `e_min_chg_pct` parameter to define the minimum price change percentage compared to the previous day's close, enhancing the momentum trading strategy. - Updated various functions and classes to incorporate this new parameter, ensuring it is utilized in both backtesting and live trading scenarios. - Improved documentation and comments to clarify the purpose and usage of the new parameter across the codebase. Impact: - This addition allows for more precise control over trading conditions, potentially increasing the effectiveness of the momentum strategy while maintaining system integrity and performance.
1122 lines
39 KiB
Python
1122 lines
39 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
optuna_web_jobs.py — 백테 웹용 Optuna 잡 (subprocess + 디스크 상태)
|
|
|
|
HTTP 타임아웃과 분리: start 는 즉시 job_id 반환, status 폴링으로 진행률/결과.
|
|
apply-best 는 절대 자동 적용하지 않음.
|
|
|
|
※ 자식 프로세스: Popen 후 반드시 wait(reaper) — 안 하면 종료 후 좀비(Z)로 남아
|
|
웹이 "이미 실행 중" 으로 막힘.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
JOBS_DIR = ROOT / "logs" / "optuna_web_jobs"
|
|
RESULTS_DIR = ROOT / "kis_trader" / "backtest" / "results"
|
|
PY = ROOT / ".venv" / "bin" / "python"
|
|
|
|
_STRATS = ("momentum", "us_momentum", "tail", "breakout", "scalp")
|
|
|
|
|
|
def _now_iso() -> str:
|
|
return datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
|
|
|
|
|
def _ensure_dirs() -> None:
|
|
JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
|
(ROOT / "logs").mkdir(parents=True, exist_ok=True)
|
|
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def _job_path(job_id: str) -> Path:
|
|
return JOBS_DIR / f"{job_id}.json"
|
|
|
|
|
|
def save_job(meta: Dict[str, Any]) -> None:
|
|
_ensure_dirs()
|
|
jid = str(meta.get("job_id") or "")
|
|
if not jid:
|
|
raise ValueError("job_id required")
|
|
path = _job_path(jid)
|
|
tmp = path.with_suffix(".tmp")
|
|
tmp.write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
tmp.replace(path)
|
|
|
|
|
|
def load_job(job_id: str) -> Optional[Dict[str, Any]]:
|
|
path = _job_path(job_id)
|
|
if not path.is_file():
|
|
return None
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _job_sort_ts(meta: Dict[str, Any], sort: str = "started") -> float:
|
|
"""
|
|
최근 잡 정렬 키. ※ 파일 mtime 금지 — refresh_job_status 가 폴링마다 save 해서
|
|
mtime 순이면 순서가 계속 뒤바뀜.
|
|
sort=started → started_ts (없으면 started_at)
|
|
sort=finished → finished_at (미종료는 맨 위, started_ts 보조)
|
|
"""
|
|
st = float(meta.get("started_ts") or 0)
|
|
if not st:
|
|
sa = str(meta.get("started_at") or "")[:19]
|
|
if sa:
|
|
try:
|
|
st = datetime.strptime(sa, "%Y-%m-%dT%H:%M:%S").timestamp()
|
|
except Exception:
|
|
st = 0.0
|
|
sort = str(sort or "started").strip().lower()
|
|
if sort != "finished":
|
|
return st
|
|
ft = float(meta.get("finished_ts") or 0)
|
|
if ft:
|
|
return ft
|
|
fa = str(meta.get("finished_at") or "")[:19]
|
|
if not fa:
|
|
# 실행 중/미종료 → 종료순에서도 최상단
|
|
return st + 1e15
|
|
try:
|
|
return datetime.strptime(fa, "%Y-%m-%dT%H:%M:%S").timestamp()
|
|
except Exception:
|
|
return st
|
|
|
|
|
|
def list_jobs(limit: int = 30, sort: str = "started") -> List[Dict[str, Any]]:
|
|
_ensure_dirs()
|
|
out: List[Dict[str, Any]] = []
|
|
for p in JOBS_DIR.glob("*.json"):
|
|
try:
|
|
out.append(json.loads(p.read_text(encoding="utf-8")))
|
|
except Exception:
|
|
continue
|
|
out.sort(key=lambda m: _job_sort_ts(m, sort), reverse=True)
|
|
return out[: max(1, int(limit))]
|
|
|
|
|
|
def _try_reap_child(pid: int) -> None:
|
|
"""웹이 부모인 좀비면 waitpid 로 회수. 아니면 ChildProcessError → 무시."""
|
|
try:
|
|
os.waitpid(int(pid), os.WNOHANG)
|
|
except (ChildProcessError, OSError, ValueError):
|
|
pass
|
|
|
|
|
|
def _pid_alive(pid: Optional[int]) -> bool:
|
|
"""프로세스가 실제로 살아 있으면 True. 좀비(Z)는 회수 후 False."""
|
|
if not pid or int(pid) <= 0:
|
|
return False
|
|
try:
|
|
os.kill(int(pid), 0)
|
|
except OSError:
|
|
return False
|
|
# Linux: /proc/<pid>/stat 상태 Z = 좀비 (부모 wait 안 함 → kill 0 은 성공)
|
|
try:
|
|
raw = Path("/proc/%d/stat" % int(pid)).read_text(encoding="utf-8", errors="replace")
|
|
rp = raw.rfind(")")
|
|
if rp >= 0 and rp + 2 < len(raw):
|
|
state = raw[rp + 2 : rp + 3]
|
|
if state == "Z":
|
|
_try_reap_child(int(pid))
|
|
return False
|
|
except Exception:
|
|
pass
|
|
return True
|
|
|
|
|
|
def _spawn_job_reaper(
|
|
proc: subprocess.Popen,
|
|
job_id: str,
|
|
log_f: Any,
|
|
) -> None:
|
|
"""
|
|
근본: Popen 자식을 wait 해서 좀비 방지 + 종료 시 job JSON 즉시 done/error 확정.
|
|
웹 프로세스가 부모로 남는 한 이 스레드가 필수.
|
|
"""
|
|
|
|
def _run() -> None:
|
|
rc: Optional[int] = None
|
|
try:
|
|
rc = int(proc.wait())
|
|
except Exception:
|
|
try:
|
|
rc = int(proc.poll()) if proc.poll() is not None else None
|
|
except Exception:
|
|
rc = None
|
|
try:
|
|
if log_f is not None and hasattr(log_f, "closed") and not log_f.closed:
|
|
log_f.flush()
|
|
log_f.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
meta = load_job(job_id)
|
|
if not meta:
|
|
return
|
|
meta["exit_code"] = rc
|
|
if rc not in (None, 0) and not meta.get("error"):
|
|
meta["error"] = "process exit_code=%s" % rc
|
|
save_job(meta)
|
|
refresh_job_status(meta)
|
|
except Exception:
|
|
pass
|
|
|
|
threading.Thread(
|
|
target=_run,
|
|
name="optuna-reap-%s" % job_id,
|
|
daemon=True,
|
|
).start()
|
|
|
|
|
|
def _tail_text(path: Optional[str], n: int = 40) -> str:
|
|
if not path:
|
|
return ""
|
|
p = Path(path)
|
|
if not p.is_file():
|
|
return ""
|
|
try:
|
|
# 큰 로그: 끝부분만
|
|
data = p.read_bytes()
|
|
if len(data) > 200_000:
|
|
data = data[-200_000:]
|
|
text = data.decode("utf-8", errors="replace")
|
|
lines = text.splitlines()
|
|
return "\n".join(lines[-n:])
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _parse_result_paths_from_log(log_path: str) -> Dict[str, Optional[str]]:
|
|
tail = _tail_text(log_path, 80)
|
|
rj = None
|
|
bm = None
|
|
m = re.search(r"OPTUNA_RESULT_JSON=(.+)", tail)
|
|
if m:
|
|
rj = m.group(1).strip()
|
|
m2 = re.search(r"OPTUNA_BRIEFING_MD=(.+)", tail)
|
|
if m2:
|
|
bm = m2.group(1).strip()
|
|
return {"result_json": rj, "briefing_md": bm}
|
|
|
|
|
|
def _study_progress(study_name: str, trials_total: int) -> Dict[str, Any]:
|
|
"""Optuna MariaDB study 기준 진행률 + best trial 실측 지표.
|
|
|
|
best_value(score)만 보여주면 의미가 안 보이므로,
|
|
best trial user_attrs 의 WR/PnL/PF/MDD/trades 도 함께 반환.
|
|
"""
|
|
out = {
|
|
"trials_done": 0,
|
|
"trials_total": int(trials_total or 0),
|
|
"pct": 0.0,
|
|
"best_value": None,
|
|
"best_trial": None,
|
|
"best_win_rate": None,
|
|
"best_pnl": None,
|
|
"best_pf": None,
|
|
"best_mdd": None,
|
|
"best_trades": None,
|
|
"study_ok": False,
|
|
}
|
|
if not study_name:
|
|
return out
|
|
try:
|
|
import optuna
|
|
from kis_trader.backtest.optuna_common import resolve_optuna_storage_url
|
|
|
|
storage = resolve_optuna_storage_url()
|
|
study = optuna.load_study(study_name=study_name, storage=storage)
|
|
n = len(study.trials)
|
|
out["trials_done"] = int(n)
|
|
tot = max(1, int(trials_total or n or 1))
|
|
out["trials_total"] = tot
|
|
out["pct"] = round(min(100.0, 100.0 * n / tot), 1)
|
|
try:
|
|
bt = study.best_trial
|
|
if bt is not None:
|
|
out["best_value"] = float(study.best_value)
|
|
out["best_trial"] = int(bt.number)
|
|
ua = bt.user_attrs or {}
|
|
if ua.get("win_rate") is not None:
|
|
out["best_win_rate"] = float(ua.get("win_rate") or 0)
|
|
if ua.get("total_pnl") is not None:
|
|
out["best_pnl"] = float(ua.get("total_pnl") or 0)
|
|
if ua.get("pf") is not None:
|
|
out["best_pf"] = float(ua.get("pf") or 0)
|
|
if ua.get("mdd") is not None:
|
|
out["best_mdd"] = float(ua.get("mdd") or 0)
|
|
if ua.get("total_trades") is not None:
|
|
out["best_trades"] = int(ua.get("total_trades") or 0)
|
|
except Exception:
|
|
out["best_value"] = None
|
|
out["study_ok"] = True
|
|
except Exception as exc:
|
|
out["error"] = str(exc)[:200]
|
|
return out
|
|
|
|
|
|
def _row_metrics(row: Optional[Dict[str, Any]], *, label: str, source: str) -> Optional[Dict[str, Any]]:
|
|
if not row:
|
|
return None
|
|
out = {
|
|
"label": label,
|
|
"source": source,
|
|
"optuna_trial_number": row.get("optuna_trial_number"),
|
|
"total_pnl": row.get("total_pnl"),
|
|
"total_trades": row.get("total_trades"),
|
|
"win_rate": row.get("win_rate"),
|
|
"pf": row.get("pf"),
|
|
"score": row.get("score"),
|
|
}
|
|
# 일별 안정성 (신규 Optuna JSON)
|
|
for k in (
|
|
"stability_score", "n_losing_days", "n_active_days",
|
|
"worst_day_pnl", "best_day_pnl", "daily_pnl_mean", "daily_pnl_std",
|
|
"daily_pnl",
|
|
):
|
|
if row.get(k) is not None:
|
|
out[k] = row.get(k)
|
|
return out
|
|
|
|
|
|
def _summarize_result_json(path: Optional[str]) -> Optional[Dict[str, Any]]:
|
|
"""완료 JSON → Top5 gated/stable · 비교표(학습1위/gated1위/stable1위/mode) · vs_best."""
|
|
if not path or not Path(path).is_file():
|
|
return None
|
|
try:
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return None
|
|
gated = list(data.get("results_gated") or [])
|
|
stable = list(data.get("results_stable") or [])
|
|
allr = list(data.get("results") or data.get("results_all") or [])
|
|
learn = allr[0] if allr else None
|
|
gate0 = gated[0] if gated else None
|
|
stab0 = stable[0] if stable else None
|
|
top = gate0 or learn
|
|
|
|
mc = data.get("mode_combo") or {}
|
|
mc_bt = mc.get("backtest") or {}
|
|
vs = mc.get("vs_best") or {}
|
|
|
|
compare_rows: List[Dict[str, Any]] = []
|
|
r_learn = _row_metrics(learn, label="학습1위(results)", source="learn")
|
|
if r_learn:
|
|
compare_rows.append(r_learn)
|
|
r_gate = _row_metrics(gate0, label="사후합격1위(gated)", source="gated")
|
|
if r_gate:
|
|
compare_rows.append(r_gate)
|
|
r_stab = _row_metrics(stab0, label="안정1위(stable)", source="stable")
|
|
if r_stab:
|
|
compare_rows.append(r_stab)
|
|
if mc_bt.get("ok") or mc_bt.get("total_pnl") is not None:
|
|
compare_rows.append({
|
|
"label": "mode_combo 실측",
|
|
"source": "mode",
|
|
"optuna_trial_number": None,
|
|
"total_pnl": mc_bt.get("total_pnl"),
|
|
"total_trades": mc_bt.get("total_trades"),
|
|
"win_rate": mc_bt.get("win_rate"),
|
|
"pf": mc_bt.get("pf"),
|
|
"score": None,
|
|
})
|
|
|
|
top5 = []
|
|
for i, row in enumerate(gated[:5], start=1):
|
|
m = _row_metrics(row, label=f"gated #{i}", source="gated")
|
|
if m:
|
|
m["rank"] = i
|
|
top5.append(m)
|
|
|
|
top5_learn: List[Dict[str, Any]] = []
|
|
for i, row in enumerate(allr[:5], start=1):
|
|
m = _row_metrics(row, label=f"learn #{i}", source="learn")
|
|
if m:
|
|
m["rank"] = i
|
|
top5_learn.append(m)
|
|
|
|
top5_stable: List[Dict[str, Any]] = []
|
|
for i, row in enumerate(stable[:5], start=1):
|
|
m = _row_metrics(row, label=f"stable #{i}", source="stable")
|
|
if m:
|
|
m["rank"] = i
|
|
top5_stable.append(m)
|
|
|
|
briefing = None
|
|
bp = str(path).replace(".json", ".briefing.md")
|
|
if Path(bp).is_file():
|
|
briefing = bp
|
|
elif data.get("briefing_md_path"):
|
|
briefing = data.get("briefing_md_path")
|
|
|
|
# 다단트레일 추천 (JSON에 없으면 재계산 — 구결과 미리보기용)
|
|
trail_rec = data.get("daily_trail_recommend")
|
|
if not isinstance(trail_rec, dict):
|
|
try:
|
|
from kis_trader.backtest.optuna_daily_trail_recommend import (
|
|
recommend_from_optuna_out_data,
|
|
)
|
|
trail_rec = recommend_from_optuna_out_data(data)
|
|
except Exception:
|
|
trail_rec = None
|
|
|
|
return {
|
|
"strategy": data.get("strategy"),
|
|
"mode": data.get("mode"),
|
|
"start": data.get("start"),
|
|
"end": data.get("end"),
|
|
"n_gated": len(gated),
|
|
"n_stable": len(stable),
|
|
"n_all": len(allr),
|
|
"optuna_best_trial_number": data.get("optuna_best_trial_number"),
|
|
"stable_gates": data.get("stable_gates"),
|
|
"top": _row_metrics(
|
|
top,
|
|
label="적용후보(gated우선)",
|
|
source="gated" if gate0 else "learn",
|
|
),
|
|
"compare_rows": compare_rows,
|
|
"vs_best": vs if vs else None,
|
|
"top5_gated": top5,
|
|
"top5_learn": top5_learn,
|
|
"top5_stable": top5_stable,
|
|
"mode_combo_note": mc.get("note"),
|
|
"daily_trail_recommend": trail_rec,
|
|
"apply_ready": bool(gate0) and float(gate0.get("total_pnl") or 0) > 0,
|
|
"apply_stable_ready": bool(stab0) and float(stab0.get("total_pnl") or 0) > 0,
|
|
"briefing_md": briefing,
|
|
}
|
|
|
|
|
|
def get_candidate_detail(
|
|
*,
|
|
job_id: Optional[str] = None,
|
|
result_json: Optional[str] = None,
|
|
source: str = "gated",
|
|
rank: int = 1,
|
|
) -> Dict[str, Any]:
|
|
"""보기용: gated/learn/mode 후보 메트릭 + params 미리보기."""
|
|
path = result_json
|
|
meta = None
|
|
if job_id:
|
|
meta = load_job(job_id)
|
|
if not meta:
|
|
raise FileNotFoundError(f"job not found: {job_id}")
|
|
path = path or meta.get("result_json")
|
|
if not path or not Path(path).is_file():
|
|
raise FileNotFoundError("result_json 없음")
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
src = str(source or "gated").strip().lower()
|
|
rank = max(1, int(rank or 1))
|
|
params: Dict[str, Any] = {}
|
|
metrics: Dict[str, Any] = {}
|
|
|
|
if src == "mode":
|
|
mc = data.get("mode_combo") or {}
|
|
params = dict(mc.get("params") or {})
|
|
bt = mc.get("backtest") or {}
|
|
metrics = {
|
|
"label": "mode_combo",
|
|
"optuna_trial_number": None,
|
|
"total_pnl": bt.get("total_pnl"),
|
|
"total_trades": bt.get("total_trades"),
|
|
"win_rate": bt.get("win_rate"),
|
|
"pf": bt.get("pf"),
|
|
}
|
|
else:
|
|
if src == "stable":
|
|
pool = data.get("results_stable")
|
|
elif src == "gated":
|
|
pool = data.get("results_gated")
|
|
else:
|
|
pool = data.get("results") or []
|
|
pool = list(pool or [])
|
|
if not pool:
|
|
raise RuntimeError(f"{src} 결과 없음")
|
|
if rank > len(pool):
|
|
raise RuntimeError(f"rank {rank} 범위 초과 (1~{len(pool)})")
|
|
item = pool[rank - 1]
|
|
try:
|
|
from kis_trader.backtest.backtest_portfolio_common import merge_param_search_apply_source
|
|
params = merge_param_search_apply_source(item, data)
|
|
except Exception:
|
|
params = dict(item.get("merged_params") or item.get("params") or {})
|
|
metrics = {
|
|
"label": f"{src} #{rank}",
|
|
"optuna_trial_number": item.get("optuna_trial_number"),
|
|
"total_pnl": item.get("total_pnl"),
|
|
"total_trades": item.get("total_trades"),
|
|
"win_rate": item.get("win_rate"),
|
|
"pf": item.get("pf"),
|
|
"score": item.get("score"),
|
|
"stability_score": item.get("stability_score"),
|
|
"n_losing_days": item.get("n_losing_days"),
|
|
"worst_day_pnl": item.get("worst_day_pnl"),
|
|
"daily_pnl": item.get("daily_pnl"),
|
|
}
|
|
|
|
# 보기: 전체 파라미터 (키 정렬). 예전 [:40] 잘림 → ratchet/rsi 등이 “없는 것처럼” 보임
|
|
preview = {k: params[k] for k in sorted(params.keys(), key=lambda x: str(x))}
|
|
return {
|
|
"ok": True,
|
|
"source": src,
|
|
"rank": rank,
|
|
"metrics": metrics,
|
|
"params_preview": preview,
|
|
"params_full": preview,
|
|
"params_count": len(params),
|
|
"result_json": str(path),
|
|
"strategy": (meta or {}).get("strategy") or data.get("strategy"),
|
|
"start": data.get("start"),
|
|
"end": data.get("end"),
|
|
}
|
|
|
|
|
|
def apply_optuna_result(
|
|
*,
|
|
job_id: Optional[str] = None,
|
|
result_json: Optional[str] = None,
|
|
source: str = "gated",
|
|
rank: int = 1,
|
|
allow_non_positive_pnl: bool = False,
|
|
symbol: Optional[str] = None,
|
|
exchange: Optional[str] = None,
|
|
stock_group: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Optuna JSON 후보 → 전략별 apply_params_to_db.
|
|
source: gated | stable | learn | mode
|
|
rank: gated/stable/learn 1-based
|
|
symbol: us_momentum 종목 cfg 적용 시 (없으면 job/JSON 메타 또는 전역)
|
|
"""
|
|
path = result_json
|
|
meta = None
|
|
if job_id:
|
|
meta = load_job(job_id)
|
|
if not meta:
|
|
raise FileNotFoundError(f"job not found: {job_id}")
|
|
if meta.get("status") != "done":
|
|
raise RuntimeError(f"job status={meta.get('status')} — 완료 후에만 적용")
|
|
path = path or meta.get("result_json")
|
|
if not path or not Path(path).is_file():
|
|
raise FileNotFoundError("result_json 없음")
|
|
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
strat = str(
|
|
(meta or {}).get("strategy")
|
|
or data.get("strategy")
|
|
or "",
|
|
).strip().lower()
|
|
if strat in ("", "all"):
|
|
# JSON 파일명에서 추론
|
|
name = Path(path).name
|
|
for s in _STRATS:
|
|
if f"_{s}_" in f"_{name}_" or name.startswith(f"optuna_{s}_"):
|
|
strat = s
|
|
break
|
|
if strat not in _STRATS:
|
|
raise ValueError(f"전략 불명: {strat}")
|
|
|
|
src = str(source or "gated").strip().lower()
|
|
rank = max(1, int(rank or 1))
|
|
item: Optional[Dict[str, Any]] = None
|
|
merged: Dict[str, Any] = {}
|
|
|
|
if src == "mode":
|
|
mc = data.get("mode_combo") or {}
|
|
merged = dict(mc.get("params") or {})
|
|
bt = mc.get("backtest") or {}
|
|
pnl = float(bt.get("total_pnl") or 0)
|
|
metrics = {
|
|
"total_pnl": bt.get("total_pnl"),
|
|
"total_trades": bt.get("total_trades"),
|
|
"win_rate": bt.get("win_rate"),
|
|
"pf": bt.get("pf"),
|
|
"optuna_trial_number": None,
|
|
}
|
|
if not merged:
|
|
raise RuntimeError("mode_combo.params 없음")
|
|
else:
|
|
if src == "stable":
|
|
pool = data.get("results_stable")
|
|
elif src == "gated":
|
|
pool = data.get("results_gated")
|
|
else:
|
|
pool = data.get("results") or []
|
|
pool = list(pool or [])
|
|
if not pool:
|
|
raise RuntimeError(f"{src} 결과 없음")
|
|
if rank > len(pool):
|
|
raise RuntimeError(f"rank {rank} 범위 초과 (1~{len(pool)})")
|
|
item = pool[rank - 1]
|
|
from kis_trader.backtest.backtest_portfolio_common import merge_param_search_apply_source
|
|
merged = merge_param_search_apply_source(item, data)
|
|
pnl = float(item.get("total_pnl") or 0)
|
|
metrics = {
|
|
"total_pnl": item.get("total_pnl"),
|
|
"total_trades": item.get("total_trades"),
|
|
"win_rate": item.get("win_rate"),
|
|
"pf": item.get("pf"),
|
|
"optuna_trial_number": item.get("optuna_trial_number"),
|
|
"stability_score": item.get("stability_score"),
|
|
"n_losing_days": item.get("n_losing_days"),
|
|
"worst_day_pnl": item.get("worst_day_pnl"),
|
|
}
|
|
|
|
if pnl <= 0 and not allow_non_positive_pnl:
|
|
raise RuntimeError(
|
|
f"total_pnl={pnl} ≤ 0 — DB 미적용. 강제 시 allow_non_positive_pnl=true",
|
|
)
|
|
|
|
sym = str(
|
|
symbol
|
|
or (meta or {}).get("symbol")
|
|
or data.get("symbol")
|
|
or data.get("_apply_symbol")
|
|
or "",
|
|
).strip().upper()
|
|
|
|
env_id = None
|
|
if strat == "momentum":
|
|
from kis_trader.backtest.param_search_momentum import apply_params_to_db
|
|
env_id = apply_params_to_db(merged)
|
|
elif strat == "us_momentum":
|
|
from kis_trader.backtest.param_search_momentum import apply_params_to_db_us
|
|
if exchange:
|
|
merged["exchange"] = str(exchange).strip() or merged.get("exchange")
|
|
if stock_group:
|
|
merged["stock_group"] = str(stock_group).strip() or merged.get("stock_group")
|
|
env_id = apply_params_to_db_us(merged, symbol=sym)
|
|
elif strat == "breakout":
|
|
from kis_trader.backtest.param_search_breakout import apply_params_to_db
|
|
apply_params_to_db(merged)
|
|
elif strat == "scalp":
|
|
from kis_trader.backtest.param_search_scalping import apply_params_to_db
|
|
apply_params_to_db(merged)
|
|
elif strat == "tail":
|
|
from kis_trader.backtest.tail_param_search import apply_params_to_db
|
|
apply_params_to_db(merged)
|
|
else:
|
|
raise ValueError(strat)
|
|
|
|
# apply 시에만 다단트레일 추천값 → 전략별 *_DAILY_PROFIT_* (탐색 축 아님)
|
|
# 종목 cfg 적용 시 전역 다단트레일 오염 금지
|
|
trail_apply: Dict[str, Any] = {"applied": False}
|
|
if not sym:
|
|
try:
|
|
from kis_trader.backtest.optuna_daily_trail_recommend import (
|
|
apply_daily_trail_recommend_from_optuna_json,
|
|
recommend_from_optuna_out_data,
|
|
)
|
|
if not data.get("daily_trail_recommend"):
|
|
data["daily_trail_recommend"] = recommend_from_optuna_out_data(data)
|
|
trail_apply = apply_daily_trail_recommend_from_optuna_json(
|
|
str(path), strategy=strat,
|
|
)
|
|
except Exception as exc:
|
|
trail_apply = {"applied": False, "error": str(exc)}
|
|
else:
|
|
trail_apply = {"applied": False, "reason": f"stock_cfg:{sym}"}
|
|
|
|
if meta is not None:
|
|
meta["applied_at"] = _now_iso()
|
|
meta["applied_source"] = src
|
|
meta["applied_rank"] = rank
|
|
meta["applied_trial"] = metrics.get("optuna_trial_number")
|
|
meta["applied_symbol"] = sym or None
|
|
meta["daily_trail_applied"] = bool(trail_apply.get("applied"))
|
|
save_job(meta)
|
|
|
|
trail_note = ""
|
|
if trail_apply.get("applied"):
|
|
patch = trail_apply.get("patch") or {}
|
|
tiers = patch.get(
|
|
next((k for k in patch if k.endswith("_DAILY_PROFIT_TRAIL_TIERS")), ""),
|
|
"",
|
|
)
|
|
trail_note = f" · 다단트레일 추천 반영 tiers={tiers}"
|
|
elif trail_apply.get("reason"):
|
|
trail_note = f" · 다단트레일 미반영({trail_apply.get('reason')})"
|
|
|
|
apply_target = f"stock_config:{sym}" if (strat == "us_momentum" and sym) else "global"
|
|
return {
|
|
"ok": True,
|
|
"strategy": strat,
|
|
"source": src,
|
|
"rank": rank,
|
|
"symbol": sym or None,
|
|
"apply_target": apply_target,
|
|
"env_id": env_id,
|
|
"metrics": metrics,
|
|
"result_json": str(path),
|
|
"daily_trail_apply": trail_apply,
|
|
"note": (
|
|
"TIME_* 는 session_env_patch 기본 OFF — 운영 시간창 유지"
|
|
+ trail_note
|
|
),
|
|
}
|
|
|
|
|
|
def register_result_json_as_job(
|
|
result_json: str,
|
|
*,
|
|
source_label: str = "cli",
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
CLI/순차 스크립트가 남긴 Optuna JSON 을 웹 잡 목록에 등록.
|
|
동일 result_json 경로가 이미 있으면 갱신만 한다.
|
|
"""
|
|
_ensure_dirs()
|
|
path = Path(result_json).resolve()
|
|
if not path.is_file():
|
|
raise FileNotFoundError(str(path))
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
strat = str(data.get("strategy") or "").strip().lower()
|
|
if strat not in _STRATS:
|
|
for s in _STRATS:
|
|
if path.name.startswith(f"optuna_{s}_"):
|
|
strat = s
|
|
break
|
|
if strat not in _STRATS:
|
|
raise ValueError(f"전략 불명: {path.name}")
|
|
|
|
brief = path.with_suffix("").as_posix()
|
|
if brief.endswith(".json"):
|
|
brief = brief[:-5]
|
|
briefing = Path(str(path).replace(".json", ".briefing.md"))
|
|
study = str(data.get("optuna_study_name") or data.get("study_name") or "")
|
|
start = str(data.get("start") or "")
|
|
end = str(data.get("end") or "")
|
|
mode = str(data.get("mode") or "tpe")
|
|
# 안정 job_id: 파일 stem
|
|
job_id = f"import_{path.stem}"
|
|
|
|
existing = load_job(job_id)
|
|
meta: Dict[str, Any] = existing or {}
|
|
meta.update({
|
|
"job_id": job_id,
|
|
"kind": "import",
|
|
"source": source_label,
|
|
"strategy": strat,
|
|
"mode": mode,
|
|
"start": start,
|
|
"end": end,
|
|
"trials": int(data.get("optuna_trials_completed") or data.get("optuna_n_trials_requested") or 0),
|
|
"study_name": study,
|
|
"status": "done",
|
|
"pid": None,
|
|
"log_path": None,
|
|
"result_json": str(path),
|
|
"briefing_md": str(briefing) if briefing.is_file() else None,
|
|
"started_ts": meta.get("started_ts") or path.stat().st_mtime,
|
|
"started_at": meta.get("started_at") or _now_iso(),
|
|
"finished_at": meta.get("finished_at") or _now_iso(),
|
|
"imported_at": _now_iso(),
|
|
})
|
|
meta["result_summary"] = _summarize_result_json(str(path))
|
|
if briefing.is_file():
|
|
try:
|
|
meta["briefing_preview"] = briefing.read_text(encoding="utf-8")[:4000]
|
|
except Exception:
|
|
pass
|
|
save_job(meta)
|
|
return meta
|
|
|
|
|
|
def import_recent_cli_results(*, limit_per_strategy: int = 3) -> List[Dict[str, Any]]:
|
|
"""전략별 최근 Optuna JSON 을 웹 잡으로 등록 (CLI 결과 노출용)."""
|
|
_ensure_dirs()
|
|
out: List[Dict[str, Any]] = []
|
|
lim = max(1, int(limit_per_strategy or 3))
|
|
for strat in _STRATS:
|
|
files = sorted(
|
|
RESULTS_DIR.glob(f"optuna_{strat}_tpe_*.json"),
|
|
key=lambda p: p.stat().st_mtime,
|
|
reverse=True,
|
|
)[:lim]
|
|
for p in files:
|
|
try:
|
|
out.append(register_result_json_as_job(str(p), source_label="cli"))
|
|
except Exception:
|
|
continue
|
|
return out
|
|
|
|
|
|
def refresh_job_status(meta: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""pid/로그/study 로 status·progress 갱신 후 저장."""
|
|
m = dict(meta)
|
|
pid = m.get("pid")
|
|
alive = _pid_alive(pid)
|
|
log_path = str(m.get("log_path") or "")
|
|
paths = _parse_result_paths_from_log(log_path) if log_path else {}
|
|
if paths.get("result_json"):
|
|
m["result_json"] = paths["result_json"]
|
|
if paths.get("briefing_md"):
|
|
m["briefing_md"] = paths["briefing_md"]
|
|
|
|
prog = _study_progress(str(m.get("study_name") or ""), int(m.get("trials") or 0))
|
|
# 순차(seq/seq4): 마스터 로그에서 현재 전략 힌트
|
|
if m.get("kind") in ("seq4", "seq") and log_path:
|
|
tail = _tail_text(log_path, 30)
|
|
mm = re.findall(r"\[(momentum|us_momentum|tail|breakout|scalp)\] START", tail)
|
|
if mm:
|
|
m["current_strategy"] = mm[-1]
|
|
if "ALL DONE" in tail:
|
|
m["status"] = "done"
|
|
alive = False
|
|
|
|
if m.get("status") not in ("done", "error"):
|
|
# 로그에 결과 JSON 이 이미 찍혔으면 좀비/고아 PID 여도 완료로 확정
|
|
rj = m.get("result_json")
|
|
if rj and Path(str(rj)).is_file() and (
|
|
"OPTUNA_RESULT_JSON=" in (_tail_text(log_path, 40) or "")
|
|
):
|
|
alive = False
|
|
if alive:
|
|
m["status"] = "running"
|
|
else:
|
|
# 프로세스 종료
|
|
if m.get("result_json") and Path(str(m["result_json"])).is_file():
|
|
m["status"] = "done"
|
|
elif m.get("kind") in ("seq4", "seq") and m.get("status") != "done":
|
|
# seq 마스터가 죽었는데 ALL DONE 없으면 error 가능
|
|
tail = _tail_text(log_path, 20)
|
|
if "ALL DONE" in (tail or ""):
|
|
m["status"] = "done"
|
|
else:
|
|
m["status"] = "error"
|
|
m["error"] = m.get("error") or "process ended without ALL DONE"
|
|
else:
|
|
# 싱글: 결과 JSON 최신 glob 보조
|
|
strat = str(m.get("strategy") or "")
|
|
if strat and strat not in ("all", "seq"):
|
|
# us_momentum 결과 파일명: optuna_us_momentum_* (구버전 momentum_* 폴백)
|
|
globs = [
|
|
f"optuna_{strat}_{m.get('mode') or 'tpe'}_*.json",
|
|
f"optuna_{strat}_*.json",
|
|
]
|
|
if strat == "us_momentum":
|
|
globs.append("optuna_momentum_tpe_*.json")
|
|
globs.append("optuna_momentum_*.json")
|
|
cands = []
|
|
for g in globs:
|
|
cands.extend(RESULTS_DIR.glob(g))
|
|
cands = sorted(
|
|
{p.resolve(): p for p in cands}.values(),
|
|
key=lambda p: p.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
if cands and cands[0].stat().st_mtime >= float(m.get("started_ts") or 0) - 5:
|
|
m["result_json"] = str(cands[0])
|
|
brief = Path(str(cands[0]).replace(".json", ".briefing.md"))
|
|
if brief.is_file():
|
|
m["briefing_md"] = str(brief)
|
|
m["status"] = "done"
|
|
else:
|
|
m["status"] = "error"
|
|
m["error"] = m.get("error") or "process ended (no result json)"
|
|
else:
|
|
m["status"] = "error"
|
|
m["error"] = m.get("error") or "process ended"
|
|
if m["status"] in ("done", "error") and not m.get("finished_at"):
|
|
m["finished_at"] = _now_iso()
|
|
m["finished_ts"] = time.time()
|
|
elif m["status"] in ("done", "error") and not m.get("finished_ts"):
|
|
fa = str(m.get("finished_at") or "")[:19]
|
|
try:
|
|
m["finished_ts"] = datetime.strptime(fa, "%Y-%m-%dT%H:%M:%S").timestamp()
|
|
except Exception:
|
|
m["finished_ts"] = float(m.get("started_ts") or time.time())
|
|
|
|
if m.get("status") == "done":
|
|
prog["pct"] = 100.0
|
|
if prog["trials_total"] and prog["trials_done"] < prog["trials_total"]:
|
|
prog["trials_done"] = prog["trials_total"]
|
|
|
|
m["progress"] = prog
|
|
m["pid_alive"] = alive
|
|
m["log_tail"] = _tail_text(log_path, 25)
|
|
m["result_summary"] = _summarize_result_json(m.get("result_json"))
|
|
if m.get("briefing_md") and Path(str(m["briefing_md"])).is_file():
|
|
try:
|
|
m["briefing_preview"] = Path(str(m["briefing_md"])).read_text(encoding="utf-8")[:4000]
|
|
except Exception:
|
|
m["briefing_preview"] = None
|
|
save_job(m)
|
|
return m
|
|
|
|
|
|
def find_running_jobs() -> List[Dict[str, Any]]:
|
|
out: List[Dict[str, Any]] = []
|
|
for meta in list_jobs(40):
|
|
if meta.get("status") == "running" or _pid_alive(meta.get("pid")):
|
|
refreshed = refresh_job_status(meta)
|
|
if refreshed.get("status") == "running":
|
|
out.append(refreshed)
|
|
return out
|
|
|
|
|
|
def any_optuna_python_running() -> Optional[Dict[str, Any]]:
|
|
"""웹 외 CLI nohup 도 상단바에 힌트용."""
|
|
try:
|
|
r = subprocess.run(
|
|
["pgrep", "-af", "param_search_optuna.py|run_optuna_4strat_tpe_seq.sh"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=3,
|
|
)
|
|
lines = []
|
|
for ln in (r.stdout or "").splitlines():
|
|
if "extglob" in ln or "pgrep" in ln:
|
|
continue
|
|
if "param_search_optuna.py" in ln or "run_optuna_4strat_tpe_seq.sh" in ln:
|
|
lines.append(ln)
|
|
if not lines:
|
|
return None
|
|
return {"external": True, "cmdline": lines[0][:240], "count": len(lines)}
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _normalize_strategies(
|
|
strategy: Optional[str] = None,
|
|
strategies: Optional[Any] = None,
|
|
) -> List[str]:
|
|
"""
|
|
웹 체크박스 / 레거시 strategy=all · 단일 문자열 → 전략 리스트.
|
|
국내4 기본(all): momentum tail breakout scalp (해외는 명시 체크 시에만).
|
|
"""
|
|
order = ("momentum", "us_momentum", "tail", "breakout", "scalp")
|
|
raw: List[str] = []
|
|
if strategies is not None:
|
|
if isinstance(strategies, str):
|
|
raw = re.split(r"[\s,]+", strategies.strip())
|
|
elif isinstance(strategies, (list, tuple)):
|
|
raw = [str(x) for x in strategies]
|
|
if not raw:
|
|
s = str(strategy or "").strip().lower()
|
|
if not s:
|
|
raise ValueError("strategy/strategies 필요")
|
|
if s in ("all", "seq", "seq4", "kr4"):
|
|
# 레거시 '4전략' = 국내 4만 (해외 자동 포함 안 함)
|
|
return ["momentum", "tail", "breakout", "scalp"]
|
|
raw = re.split(r"[\s,]+", s)
|
|
seen = set()
|
|
out: List[str] = []
|
|
for x in raw:
|
|
k = str(x or "").strip().lower()
|
|
if not k or k in seen:
|
|
continue
|
|
if k not in _STRATS:
|
|
raise ValueError(f"unknown strategy={k} (허용: {_STRATS})")
|
|
seen.add(k)
|
|
out.append(k)
|
|
if not out:
|
|
raise ValueError("전략을 1개 이상 선택하세요")
|
|
# 표시·실행 순서는 고정 순서(체크 순 혼선 방지)
|
|
return [k for k in order if k in seen]
|
|
|
|
|
|
def start_optuna_job(
|
|
*,
|
|
strategy: Optional[str] = None,
|
|
strategies: Optional[Any] = None,
|
|
start: str,
|
|
end: str,
|
|
trials: int = 200,
|
|
mode: str = "tpe",
|
|
symbol: Optional[str] = None,
|
|
universe_history_source: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
subprocess 로 Optuna 시작. apply-best 없음.
|
|
전략 2개 이상 → scripts/run_optuna_4strat_tpe_seq.sh + STRATEGIES=
|
|
(레거시 strategy='all' → 국내 4순차, 해외 미포함)
|
|
symbol: us_momentum 종목 cfg Optuna (1종목 유니버스). 순차잡과 병행 불가.
|
|
universe_history_source: kiwoom|ls (저장 후보 이력 테이블).
|
|
"""
|
|
_ensure_dirs()
|
|
running = find_running_jobs()
|
|
if running:
|
|
raise RuntimeError(
|
|
f"이미 실행 중 job={running[0].get('job_id')} "
|
|
f"({running[0].get('strategy')}). 끝난 뒤 다시 시작하세요."
|
|
)
|
|
ext = any_optuna_python_running()
|
|
if ext:
|
|
raise RuntimeError(
|
|
"CLI/다른 Optuna 프로세스가 이미 실행 중입니다. "
|
|
"끝난 뒤 웹에서 시작하세요. (" + str(ext.get("cmdline") or "")[:120] + ")"
|
|
)
|
|
|
|
picked = _normalize_strategies(strategy=strategy, strategies=strategies)
|
|
mode = str(mode or "tpe").strip().lower() or "tpe"
|
|
trials = max(1, min(2000, int(trials or 200)))
|
|
start = str(start or "").strip()
|
|
end = str(end or "").strip()
|
|
if not start or not end:
|
|
raise ValueError("start/end 필요")
|
|
sym = str(symbol or "").strip().upper()
|
|
if sym:
|
|
if len(picked) != 1 or picked[0] != "us_momentum":
|
|
raise ValueError("종목 Optuna(--symbol)는 us_momentum 단독만 가능")
|
|
|
|
from kis_trader.backtest.universe_history_source import (
|
|
resolve_backtest_universe_history_source,
|
|
)
|
|
from kis_trader.utils.kr_trading_day import clamp_to_prev_kr_trading_day
|
|
|
|
hist_src = resolve_backtest_universe_history_source(universe_history_source)
|
|
start = clamp_to_prev_kr_trading_day(start)
|
|
end = clamp_to_prev_kr_trading_day(end)
|
|
if start > end:
|
|
start, end = end, start
|
|
|
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
started_ts = time.time()
|
|
|
|
env = os.environ.copy()
|
|
env["PYTHONUNBUFFERED"] = "1"
|
|
env["BACKTEST_UNIVERSE_HISTORY_SOURCE"] = hist_src
|
|
|
|
_labels = {
|
|
"momentum": "모멘텀",
|
|
"us_momentum": "해외모멘텀",
|
|
"tail": "꼬리",
|
|
"breakout": "돌파",
|
|
"scalp": "스캘핑",
|
|
}
|
|
|
|
if len(picked) >= 2:
|
|
job_id = f"opt_{ts}_seq"
|
|
log_path = ROOT / "logs" / f"optuna_web_seq_{ts}.log"
|
|
study_name = f"seq_{start.replace('-', '')}_{end.replace('-', '')}_{ts}"
|
|
cmd = [
|
|
"bash",
|
|
str(ROOT / "scripts" / "run_optuna_4strat_tpe_seq.sh"),
|
|
]
|
|
env["START"] = start
|
|
env["END"] = end
|
|
env["TRIALS"] = str(trials)
|
|
env["MODE"] = mode
|
|
env["MIN_WIN_RATE"] = "0"
|
|
env["MIN_PF"] = "0"
|
|
env["MIN_TRADES"] = "1"
|
|
env["STRATEGIES"] = " ".join(picked)
|
|
env["UNIVERSE_HISTORY_SOURCE"] = hist_src
|
|
kind = "seq"
|
|
label = "순차(" + "+".join(_labels.get(s, s) for s in picked) + ")"
|
|
strat_field = ",".join(picked)
|
|
else:
|
|
strat = picked[0]
|
|
job_id = f"opt_{ts}_{strat[:4]}"
|
|
if sym and strat == "us_momentum":
|
|
study_name = (
|
|
f"usmom_{sym}_{mode}_{start.replace('-', '')}_{end.replace('-', '')}_{ts}"
|
|
)
|
|
log_path = ROOT / "logs" / f"optuna_web_usmom_{sym}_{ts}.log"
|
|
label = f"해외모멘텀·종목 {sym}"
|
|
else:
|
|
study_name = f"{strat}_{mode}_{start.replace('-', '')}_{end.replace('-', '')}_{ts}"
|
|
log_path = ROOT / "logs" / f"optuna_web_{strat}_{ts}.log"
|
|
label = _labels.get(strat, strat)
|
|
sort_by = "score" if strat in ("momentum", "us_momentum", "scalp") else "pnl"
|
|
cmd = [
|
|
str(PY if PY.is_file() else "python3"),
|
|
"-u",
|
|
str(ROOT / "kis_trader" / "backtest" / "param_search_optuna.py"),
|
|
"--strategy", strat,
|
|
"--mode", mode,
|
|
"--start", start,
|
|
"--end", end,
|
|
"--trials", str(trials),
|
|
"--min_trades", "1",
|
|
"--min_win_rate", "0",
|
|
"--min_pf", "0",
|
|
"--orderbook-filter", "off",
|
|
"--no-progress",
|
|
"--study-name", study_name,
|
|
"--sort-by", sort_by,
|
|
"--universe-history-source", hist_src,
|
|
]
|
|
if sym and strat == "us_momentum":
|
|
cmd.extend(["--symbol", sym])
|
|
kind = "single"
|
|
strat_field = strat
|
|
|
|
log_f = open(log_path, "w", encoding="utf-8")
|
|
# start_new_session: 세션 분리. 부모 wait 필수(reaper) — 없으면 좀비(Z).
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
cwd=str(ROOT),
|
|
env=env,
|
|
stdout=log_f,
|
|
stderr=subprocess.STDOUT,
|
|
start_new_session=True,
|
|
)
|
|
meta = {
|
|
"job_id": job_id,
|
|
"kind": kind,
|
|
"label": label,
|
|
"strategy": strat_field,
|
|
"strategies": picked,
|
|
"mode": mode,
|
|
"start": start,
|
|
"end": end,
|
|
"trials": trials,
|
|
"study_name": study_name,
|
|
"symbol": sym or None,
|
|
"universe_history_source": hist_src,
|
|
"log_path": str(log_path),
|
|
"pid": int(proc.pid),
|
|
"status": "running",
|
|
"started_at": _now_iso(),
|
|
"started_ts": started_ts,
|
|
"finished_at": None,
|
|
"finished_ts": None,
|
|
"result_json": None,
|
|
"briefing_md": None,
|
|
"error": None,
|
|
"apply_best": False,
|
|
"cmd": " ".join(cmd)[:500],
|
|
}
|
|
save_job(meta)
|
|
_spawn_job_reaper(proc, job_id, log_f)
|
|
# latest pointer
|
|
(ROOT / "logs" / "optuna_web_latest_job.txt").write_text(job_id + "\n", encoding="utf-8")
|
|
return refresh_job_status(meta)
|
|
|
|
|
|
def stop_optuna_job(job_id: str) -> Dict[str, Any]:
|
|
"""프로세스 그룹 kill (선택). 결과는 보장하지 않음."""
|
|
meta = load_job(job_id)
|
|
if not meta:
|
|
raise FileNotFoundError(job_id)
|
|
pid = int(meta.get("pid") or 0)
|
|
if pid and _pid_alive(pid):
|
|
try:
|
|
os.killpg(pid, signal.SIGTERM)
|
|
except Exception:
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
except Exception as exc:
|
|
meta["error"] = str(exc)
|
|
meta["status"] = "error"
|
|
meta["error"] = meta.get("error") or "stopped by user"
|
|
meta["finished_at"] = _now_iso()
|
|
save_job(meta)
|
|
return refresh_job_status(meta)
|