673 lines
24 KiB
Python
673 lines
24 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
kis_trader/backtest/optuna_common.py — Optuna storage·DB 공통 (MariaDB 141)
|
||
=========================================================================
|
||
TradeDB(database.py) 와 동일 호스트·계정, 전용 DB kis_optuna 에 study 저장.
|
||
Win11·VM 양쪽에서 같은 storage 로 trial 공유·재개 가능.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import os
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
from urllib.parse import quote_plus
|
||
|
||
from kis_trader.utils.env import get_env_float, get_env_from_db, get_env_int
|
||
|
||
logger = logging.getLogger("optuna_common")
|
||
|
||
# Optuna 전용 MariaDB (매매 DB kis_quant_db 와 분리)
|
||
DEFAULT_OPTUNA_DB_NAME = "kis_optuna"
|
||
OPTUNA_STRATEGIES = ("tail", "momentum", "us_momentum", "breakout", "scalp", "dart")
|
||
|
||
# 탐색(TPE 학습): 게이트 OFF(0) — PnL 차이를 샘플러가 보도록.
|
||
# 리포트/apply 후보: 아래 REPORT_* 로 사후 필터.
|
||
OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT = 0.0
|
||
OPTUNA_SEARCH_MIN_PF_DEFAULT = 0.0
|
||
OPTUNA_SEARCH_MIN_TRADES_DEFAULT = 1
|
||
OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT = 40.0
|
||
OPTUNA_REPORT_MIN_PF_DEFAULT = 1.0
|
||
|
||
# 일별 손익 안정성 티어 (results_stable) — 학습1위/gated 와 별도 후보
|
||
# 손실일·최악일·일PnL 분산으로 “합산만 큰” 후보를 걸러낸다.
|
||
OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT = 1
|
||
OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT = -30000.0
|
||
OPTUNA_STABLE_LAMBDA_DEFAULT = 1.0
|
||
OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT = 2
|
||
|
||
|
||
def optuna_search_gate_defaults() -> Tuple[float, float, int]:
|
||
"""탐색 중 objective 게이트 (기본 0/0/1). CLI 미지정 시 사용."""
|
||
return (
|
||
float(get_env_float("PARAM_SEARCH_OPTUNA_MIN_WIN_RATE", OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT)),
|
||
float(get_env_float("PARAM_SEARCH_OPTUNA_MIN_PF", OPTUNA_SEARCH_MIN_PF_DEFAULT)),
|
||
int(get_env_int("PARAM_SEARCH_OPTUNA_MIN_TRADES", OPTUNA_SEARCH_MIN_TRADES_DEFAULT)),
|
||
)
|
||
|
||
|
||
def optuna_report_gate_defaults() -> Tuple[float, float, int]:
|
||
"""결과 후보·apply 사후 필터 (기본 승률40·PF1.0·min_trades=탐색과 동일)."""
|
||
_sw, _sp, min_tr = optuna_search_gate_defaults()
|
||
return (
|
||
float(get_env_float(
|
||
"PARAM_SEARCH_OPTUNA_REPORT_MIN_WIN_RATE", OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT,
|
||
)),
|
||
float(get_env_float(
|
||
"PARAM_SEARCH_OPTUNA_REPORT_MIN_PF", OPTUNA_REPORT_MIN_PF_DEFAULT,
|
||
)),
|
||
int(get_env_int("PARAM_SEARCH_OPTUNA_REPORT_MIN_TRADES", max(1, min_tr))),
|
||
)
|
||
|
||
|
||
def _sort_optuna_rows(rows: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]:
|
||
sb = (sort_by or "pnl").strip().lower()
|
||
out = list(rows)
|
||
|
||
def _f(r: Dict[str, Any], k: str) -> float:
|
||
try:
|
||
return float(r.get(k) or 0)
|
||
except (TypeError, ValueError):
|
||
return 0.0
|
||
|
||
if sb == "score":
|
||
out.sort(key=lambda r: (-_f(r, "score"), -_f(r, "total_pnl"), -_f(r, "win_rate")))
|
||
elif sb == "win_rate":
|
||
out.sort(key=lambda r: (-_f(r, "win_rate"), -_f(r, "total_pnl")))
|
||
elif sb in ("stability", "stable"):
|
||
# 일평균 − λ·표준편차(stability_score) 우선 · 최악일 · 합산 PnL
|
||
out.sort(
|
||
key=lambda r: (
|
||
-_f(r, "stability_score"),
|
||
-_f(r, "worst_day_pnl"),
|
||
-_f(r, "total_pnl"),
|
||
-_f(r, "win_rate"),
|
||
),
|
||
)
|
||
else:
|
||
out.sort(key=lambda r: (-_f(r, "total_pnl"), -_f(r, "win_rate")))
|
||
return out
|
||
|
||
|
||
def trade_exit_day_key(trade: Dict[str, Any]) -> str:
|
||
"""청산 시각 → YYYY-MM-DD (없으면 빈 문자열).
|
||
|
||
꼬리 백테는 exit_time, 스캘핑·모멘텀·돌파 포트폴리오 백테는 sell_time 을 씀.
|
||
sell_time 누락 시 daily_pnl/results_stable 이 전부 비게 됨.
|
||
"""
|
||
raw = (
|
||
trade.get("exit_time")
|
||
or trade.get("sell_date")
|
||
or trade.get("sell_time") # scalp/momentum/breakout 포트폴리오
|
||
or trade.get("exit_ts")
|
||
or trade.get("exit_at")
|
||
or ""
|
||
)
|
||
s = str(raw).strip()
|
||
if not s:
|
||
return ""
|
||
digits = "".join(ch for ch in s if ch.isdigit())
|
||
if len(digits) >= 8:
|
||
return f"{digits[0:4]}-{digits[4:6]}-{digits[6:8]}"
|
||
if len(s) >= 10 and s[4] == "-" and s[7] == "-":
|
||
return s[:10]
|
||
return ""
|
||
|
||
|
||
def compute_daily_stability_metrics(
|
||
trades: List[Dict[str, Any]],
|
||
*,
|
||
stability_lambda: Optional[float] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
거래 리스트 → 일별 PnL·안정성 점수.
|
||
|
||
stability_score = mean(일PnL) − λ × std(일PnL)
|
||
(λ 기본 OPTUNA_STABLE_LAMBDA / get_env)
|
||
"""
|
||
from statistics import mean, pstdev
|
||
|
||
if stability_lambda is None:
|
||
_, _, lam, _ = optuna_stable_gate_defaults()
|
||
stability_lambda = lam
|
||
try:
|
||
lam = float(stability_lambda)
|
||
except (TypeError, ValueError):
|
||
lam = float(OPTUNA_STABLE_LAMBDA_DEFAULT)
|
||
|
||
by_day: Dict[str, float] = {}
|
||
for t in trades or []:
|
||
day = trade_exit_day_key(t if isinstance(t, dict) else {})
|
||
if not day:
|
||
continue
|
||
try:
|
||
pnl = float((t or {}).get("pnl") or (t or {}).get("realized_pnl") or 0)
|
||
except (TypeError, ValueError):
|
||
pnl = 0.0
|
||
by_day[day] = by_day.get(day, 0.0) + pnl
|
||
|
||
days_sorted = sorted(by_day.keys())
|
||
vals = [float(by_day[d]) for d in days_sorted]
|
||
n_days = len(vals)
|
||
if n_days <= 0:
|
||
return {
|
||
"daily_pnl": {},
|
||
"n_active_days": 0,
|
||
"n_losing_days": 0,
|
||
"worst_day_pnl": 0.0,
|
||
"best_day_pnl": 0.0,
|
||
"daily_pnl_mean": 0.0,
|
||
"daily_pnl_std": 0.0,
|
||
"stability_score": 0.0,
|
||
"stability_lambda": lam,
|
||
}
|
||
|
||
n_lose = sum(1 for v in vals if v < 0)
|
||
worst = min(vals)
|
||
best = max(vals)
|
||
avg = float(mean(vals))
|
||
std = float(pstdev(vals)) if n_days >= 2 else 0.0
|
||
score = avg - lam * std
|
||
return {
|
||
"daily_pnl": {d: round(by_day[d], 2) for d in days_sorted},
|
||
"n_active_days": n_days,
|
||
"n_losing_days": int(n_lose),
|
||
"worst_day_pnl": round(worst, 2),
|
||
"best_day_pnl": round(best, 2),
|
||
"daily_pnl_mean": round(avg, 2),
|
||
"daily_pnl_std": round(std, 2),
|
||
"stability_score": round(score, 4),
|
||
"stability_lambda": lam,
|
||
}
|
||
|
||
|
||
def attach_daily_stability(
|
||
result: Dict[str, Any],
|
||
trades: List[Dict[str, Any]],
|
||
) -> Dict[str, Any]:
|
||
"""evaluate_* 반환 dict 에 일별 안정성 필드를 붙인다."""
|
||
if not isinstance(result, dict):
|
||
return result
|
||
result.update(compute_daily_stability_metrics(trades or []))
|
||
return result
|
||
|
||
|
||
def optuna_stable_gate_defaults() -> Tuple[int, float, float, int]:
|
||
"""(max_losing_days, min_worst_day_pnl, lambda, min_active_days)."""
|
||
return (
|
||
int(get_env_int(
|
||
"PARAM_SEARCH_OPTUNA_STABLE_MAX_LOSING_DAYS",
|
||
OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT,
|
||
)),
|
||
float(get_env_float(
|
||
"PARAM_SEARCH_OPTUNA_STABLE_MIN_WORST_DAY_PNL",
|
||
OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT,
|
||
)),
|
||
float(get_env_float(
|
||
"PARAM_SEARCH_OPTUNA_STABLE_LAMBDA",
|
||
OPTUNA_STABLE_LAMBDA_DEFAULT,
|
||
)),
|
||
int(get_env_int(
|
||
"PARAM_SEARCH_OPTUNA_STABLE_MIN_ACTIVE_DAYS",
|
||
OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT,
|
||
)),
|
||
)
|
||
|
||
|
||
def row_passes_report_gates(
|
||
row: Dict[str, Any],
|
||
*,
|
||
min_win_rate: float,
|
||
min_pf: float,
|
||
min_trades: int,
|
||
) -> bool:
|
||
try:
|
||
wr = float(row.get("win_rate") or 0)
|
||
pf = float(row.get("pf") or 0)
|
||
nt = int(row.get("total_trades") or 0)
|
||
except (TypeError, ValueError):
|
||
return False
|
||
if nt < int(min_trades):
|
||
return False
|
||
if wr < float(min_win_rate):
|
||
return False
|
||
if pf < float(min_pf):
|
||
return False
|
||
return True
|
||
|
||
|
||
def row_passes_stable_gates(row: Dict[str, Any]) -> bool:
|
||
"""
|
||
일별 안정성 사후 게이트.
|
||
daily_pnl / n_active_days 가 없으면(구 JSON) 통과 불가 → results_stable 빈 목록.
|
||
"""
|
||
if row.get("daily_pnl") is None and row.get("n_active_days") is None:
|
||
return False
|
||
max_lose, min_worst, _lam, min_days = optuna_stable_gate_defaults()
|
||
try:
|
||
n_days = int(row.get("n_active_days") or 0)
|
||
n_lose = int(row.get("n_losing_days") or 0)
|
||
worst = float(row.get("worst_day_pnl") or 0)
|
||
except (TypeError, ValueError):
|
||
return False
|
||
if n_days < int(min_days):
|
||
return False
|
||
if n_lose > int(max_lose):
|
||
return False
|
||
if worst < float(min_worst):
|
||
return False
|
||
return True
|
||
|
||
|
||
def set_optuna_trial_stability_attrs(trial: Any, result: Dict[str, Any]) -> None:
|
||
"""Optuna trial.user_attrs 에 일별 안정성 스냅샷 저장."""
|
||
import json as _json
|
||
|
||
if not result:
|
||
return
|
||
try:
|
||
trial.set_user_attr("n_active_days", int(result.get("n_active_days") or 0))
|
||
trial.set_user_attr("n_losing_days", int(result.get("n_losing_days") or 0))
|
||
trial.set_user_attr("worst_day_pnl", float(result.get("worst_day_pnl") or 0))
|
||
trial.set_user_attr("best_day_pnl", float(result.get("best_day_pnl") or 0))
|
||
trial.set_user_attr("daily_pnl_mean", float(result.get("daily_pnl_mean") or 0))
|
||
trial.set_user_attr("daily_pnl_std", float(result.get("daily_pnl_std") or 0))
|
||
trial.set_user_attr("stability_score", float(result.get("stability_score") or 0))
|
||
trial.set_user_attr(
|
||
"daily_pnl_json",
|
||
_json.dumps(result.get("daily_pnl") or {}, ensure_ascii=False),
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def stability_fields_from_trial_attrs(trial: Any) -> Dict[str, Any]:
|
||
"""trial.user_attrs → 결과 row 안정성 필드."""
|
||
import json as _json
|
||
|
||
raw = trial.user_attrs.get("daily_pnl_json") or "{}"
|
||
try:
|
||
daily = _json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||
except Exception:
|
||
daily = {}
|
||
if trial.user_attrs.get("n_active_days") is None and not daily:
|
||
return {}
|
||
return {
|
||
"daily_pnl": daily if isinstance(daily, dict) else {},
|
||
"n_active_days": int(trial.user_attrs.get("n_active_days") or 0),
|
||
"n_losing_days": int(trial.user_attrs.get("n_losing_days") or 0),
|
||
"worst_day_pnl": float(trial.user_attrs.get("worst_day_pnl") or 0),
|
||
"best_day_pnl": float(trial.user_attrs.get("best_day_pnl") or 0),
|
||
"daily_pnl_mean": float(trial.user_attrs.get("daily_pnl_mean") or 0),
|
||
"daily_pnl_std": float(trial.user_attrs.get("daily_pnl_std") or 0),
|
||
"stability_score": float(trial.user_attrs.get("stability_score") or 0),
|
||
}
|
||
|
||
|
||
def build_optuna_result_tiers(
|
||
rows: List[Dict[str, Any]],
|
||
*,
|
||
sort_by: str,
|
||
top_n: int = 5000,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
탐색 전체 vs 리포트/apply 후보 분리.
|
||
|
||
- results_all: 완료·게이트통과(탐색게이트) trial 전부 정렬
|
||
- results: 하위호환 — 플러스 PnL 우선(없으면 all)
|
||
- results_gated: 승률·PF 사후 필터 (apply 후보)
|
||
- results_stable: gated ∩ 일별 안정성 게이트 (들쭉날쭉 완화 후보)
|
||
"""
|
||
rep_wr, rep_pf, rep_tr = optuna_report_gate_defaults()
|
||
max_lose, min_worst, lam, min_days = optuna_stable_gate_defaults()
|
||
all_sorted = _sort_optuna_rows(rows, sort_by)
|
||
profitable = [r for r in all_sorted if float(r.get("total_pnl") or 0) > 0]
|
||
learning = profitable if profitable else all_sorted
|
||
gated = [
|
||
r for r in all_sorted
|
||
if row_passes_report_gates(
|
||
r, min_win_rate=rep_wr, min_pf=rep_pf, min_trades=rep_tr,
|
||
)
|
||
and float(r.get("total_pnl") or 0) > 0
|
||
]
|
||
stable_pool = [r for r in gated if row_passes_stable_gates(r)]
|
||
stable = _sort_optuna_rows(stable_pool, "stability")
|
||
return {
|
||
"results_all": all_sorted[:top_n],
|
||
"results": learning[:top_n],
|
||
"results_gated": gated[:top_n],
|
||
"results_stable": stable[:top_n],
|
||
"report_gates": {
|
||
"min_win_rate": rep_wr,
|
||
"min_pf": rep_pf,
|
||
"min_trades": rep_tr,
|
||
},
|
||
"stable_gates": {
|
||
"max_losing_days": max_lose,
|
||
"min_worst_day_pnl": min_worst,
|
||
"stability_lambda": lam,
|
||
"min_active_days": min_days,
|
||
"score_note": "stability_score = mean(일PnL) − λ × std(일PnL)",
|
||
},
|
||
"search_gates_note": (
|
||
"탐색 min_win_rate/min_pf 기본 0 — TPE가 PnL 차이를 학습. "
|
||
"적용·운영 후보는 results_gated(report_gates). "
|
||
"들쭉날쭉 완화 후보는 results_stable(stable_gates)."
|
||
),
|
||
"n_results_all": len(all_sorted),
|
||
"n_results_learning": len(learning),
|
||
"n_results_gated": len(gated),
|
||
"n_results_stable": len(stable),
|
||
}
|
||
|
||
|
||
def pick_gated_apply_trial(
|
||
study: Any,
|
||
*,
|
||
sort_by: str = "pnl",
|
||
fail_objective: float = -1e18,
|
||
) -> Optional[Any]:
|
||
"""
|
||
--apply-best 용: study.best(탐색 objective)가 아니라
|
||
report_gates 통과 trial 중 정렬 1위.
|
||
"""
|
||
import optuna # noqa: WPS433 — 호출 시에만
|
||
|
||
rep_wr, rep_pf, rep_tr = optuna_report_gate_defaults()
|
||
cand: List[Tuple[Dict[str, Any], Any]] = []
|
||
for trial in study.trials:
|
||
if trial.state != optuna.trial.TrialState.COMPLETE:
|
||
continue
|
||
if not trial.user_attrs.get("gates_ok"):
|
||
continue
|
||
try:
|
||
val = float(trial.value) if trial.value is not None else fail_objective
|
||
except (TypeError, ValueError):
|
||
val = fail_objective
|
||
if val <= fail_objective + 1:
|
||
continue
|
||
row = {
|
||
"win_rate": float(trial.user_attrs.get("win_rate") or 0),
|
||
"pf": float(trial.user_attrs.get("pf") or 0),
|
||
"total_trades": int(trial.user_attrs.get("total_trades") or 0),
|
||
"total_pnl": float(trial.user_attrs.get("total_pnl") or 0),
|
||
"score": float(trial.user_attrs.get("score") or 0),
|
||
"_trial_number": int(trial.number),
|
||
}
|
||
if not row_passes_report_gates(
|
||
row, min_win_rate=rep_wr, min_pf=rep_pf, min_trades=rep_tr,
|
||
):
|
||
continue
|
||
if float(row["total_pnl"]) <= 0:
|
||
continue
|
||
cand.append((row, trial))
|
||
if not cand:
|
||
return None
|
||
ranked = _sort_optuna_rows([r for r, _ in cand], sort_by)
|
||
top_n = int(ranked[0].get("_trial_number") or -1)
|
||
for r, t in cand:
|
||
if int(r.get("_trial_number") or -2) == top_n:
|
||
return t
|
||
return cand[0][1]
|
||
|
||
|
||
def ensure_optuna_gate_env_defaults(db: Any = None) -> None:
|
||
"""신규 Optuna 게이트 키가 DB에 없으면 env_config_ext 에만 UPSERT (전체 스냅샷 X)."""
|
||
defaults = {
|
||
"PARAM_SEARCH_OPTUNA_MIN_WIN_RATE": str(OPTUNA_SEARCH_MIN_WIN_RATE_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_MIN_PF": str(OPTUNA_SEARCH_MIN_PF_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_MIN_TRADES": str(OPTUNA_SEARCH_MIN_TRADES_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_REPORT_MIN_WIN_RATE": str(OPTUNA_REPORT_MIN_WIN_RATE_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_REPORT_MIN_PF": str(OPTUNA_REPORT_MIN_PF_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_BRIEFING_AI": "1",
|
||
# 일별 안정성 티어 (results_stable)
|
||
"PARAM_SEARCH_OPTUNA_STABLE_MAX_LOSING_DAYS": str(OPTUNA_STABLE_MAX_LOSING_DAYS_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_STABLE_MIN_WORST_DAY_PNL": str(OPTUNA_STABLE_MIN_WORST_DAY_PNL_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_STABLE_LAMBDA": str(OPTUNA_STABLE_LAMBDA_DEFAULT),
|
||
"PARAM_SEARCH_OPTUNA_STABLE_MIN_ACTIVE_DAYS": str(OPTUNA_STABLE_MIN_ACTIVE_DAYS_DEFAULT),
|
||
# Optuna apply 시 다단트레일 추천 → 전략별 *_DAILY_PROFIT_* (탐색 축 아님)
|
||
"OPTUNA_DAILY_TRAIL_APPLY_ON_BEST": "true",
|
||
"OPTUNA_DAILY_TRAIL_ARM_FRAC": "0.60",
|
||
"OPTUNA_DAILY_TRAIL_BEST_FRAC": "0.70",
|
||
"OPTUNA_DAILY_TRAIL_ARM_STEP": "5000",
|
||
"OPTUNA_DAILY_TRAIL_MIN_ARM": "10000",
|
||
"OPTUNA_DAILY_TRAIL_TIER_DROPS": "40,30,20",
|
||
}
|
||
try:
|
||
from datetime import datetime
|
||
|
||
from database import TradeDB
|
||
except ImportError:
|
||
return
|
||
owned = False
|
||
if db is None:
|
||
db = TradeDB()
|
||
owned = True
|
||
try:
|
||
snap = db.get_merged_env_snapshot() or {}
|
||
patch = {}
|
||
for k, v in defaults.items():
|
||
cur = snap.get(k)
|
||
if cur is None or str(cur).strip() == "":
|
||
patch[k] = v
|
||
if not patch:
|
||
return
|
||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
n = db._persist_env_config_overflow(patch, now)
|
||
try:
|
||
from kis_trader.utils.env import invalidate_merged_env_cache
|
||
invalidate_merged_env_cache()
|
||
except Exception:
|
||
pass
|
||
logger.info(
|
||
"📌 Optuna 게이트 기본값 DB(ext) 반영 %d키: %s",
|
||
n, sorted(patch.keys()),
|
||
)
|
||
except Exception as exc:
|
||
logger.warning("⚠️ Optuna 게이트 기본값 DB 반영 실패: %s", exc)
|
||
finally:
|
||
if owned:
|
||
try:
|
||
db.conn.close()
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def mariadb_creds() -> dict:
|
||
"""TradeDB(database.py) 와 동일 우선순위 — env > 기본 141."""
|
||
return {
|
||
"host": os.environ.get("DB_HOST", "192.168.0.141"),
|
||
"port": int(os.environ.get("DB_PORT", "3306")),
|
||
"user": os.environ.get("DB_USER", "jae"),
|
||
"password": os.environ.get("DB_PASS", "1234"),
|
||
}
|
||
|
||
|
||
def resolve_optuna_db_name() -> str:
|
||
"""
|
||
Optuna storage DB — 기본 kis_optuna (매매 kis_quant_db 와 분리).
|
||
env OPTUNA_DB_NAME 로 오버라이드 가능.
|
||
"""
|
||
raw = get_env_from_db("OPTUNA_DB_NAME", "")
|
||
if raw and str(raw).strip() not in ("", "None"):
|
||
return str(raw).strip()
|
||
env = os.environ.get("OPTUNA_DB_NAME", "")
|
||
if env and str(env).strip():
|
||
return str(env).strip()
|
||
return DEFAULT_OPTUNA_DB_NAME
|
||
|
||
|
||
def build_mariadb_storage_url(db_name: Optional[str] = None) -> str:
|
||
"""mysql+pymysql://…@141/optuna 형식 storage URL."""
|
||
creds = mariadb_creds()
|
||
name = (db_name or resolve_optuna_db_name()).strip()
|
||
user = quote_plus(creds["user"])
|
||
passwd = quote_plus(creds["password"])
|
||
return (
|
||
f"mysql+pymysql://{user}:{passwd}@{creds['host']}:{creds['port']}/{name}"
|
||
f"?charset=utf8mb4"
|
||
)
|
||
|
||
|
||
def ensure_optuna_database(db_name: Optional[str] = None) -> str:
|
||
"""
|
||
MariaDB 141 — kis_optuna 존재 확인 (없으면 CREATE 시도).
|
||
"""
|
||
name = (db_name or resolve_optuna_db_name()).strip()
|
||
creds = mariadb_creds()
|
||
|
||
try:
|
||
import pymysql
|
||
except ImportError as exc:
|
||
raise ImportError(
|
||
"Optuna MariaDB storage 는 pymysql 필요: pip install PyMySQL"
|
||
) from exc
|
||
|
||
# DB 존재 여부만 확인 (이미 있으면 CREATE 생략)
|
||
conn = pymysql.connect(
|
||
host=creds["host"],
|
||
port=creds["port"],
|
||
user=creds["user"],
|
||
password=creds["password"],
|
||
charset="utf8mb4",
|
||
autocommit=True,
|
||
connect_timeout=10,
|
||
)
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute("SHOW DATABASES LIKE %s", (name,))
|
||
exists = cur.fetchone() is not None
|
||
if not exists:
|
||
cur.execute(
|
||
f"CREATE DATABASE IF NOT EXISTS `{name}` "
|
||
"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
||
)
|
||
logger.info(
|
||
"📦 Optuna DB 생성: %s@%s:%s/%s",
|
||
creds["user"], creds["host"], creds["port"], name,
|
||
)
|
||
else:
|
||
logger.info(
|
||
"📦 Optuna storage DB: %s@%s:%s/%s",
|
||
creds["user"], creds["host"], creds["port"], name,
|
||
)
|
||
except Exception as exc:
|
||
logger.error("❌ Optuna DB '%s' 접속/확인 실패: %s", name, exc)
|
||
raise
|
||
finally:
|
||
conn.close()
|
||
return name
|
||
|
||
|
||
def resolve_optuna_storage_url(cli_override: Optional[str] = None) -> str:
|
||
"""
|
||
Storage URL 우선순위:
|
||
1) CLI --storage
|
||
2) OPTUNA_STORAGE_URL (DB/env)
|
||
3) MariaDB 141 / kis_optuna (TradeDB 동일 계정)
|
||
"""
|
||
if cli_override and str(cli_override).strip():
|
||
return str(cli_override).strip()
|
||
from_db = get_env_from_db("OPTUNA_STORAGE_URL", "")
|
||
if from_db and str(from_db).strip() not in ("", "None"):
|
||
return str(from_db).strip()
|
||
db_name = ensure_optuna_database()
|
||
return build_mariadb_storage_url(db_name)
|
||
|
||
|
||
def resolve_study_name(
|
||
*,
|
||
strategy: str,
|
||
mode: str,
|
||
start: str,
|
||
end: str,
|
||
cli_override: Optional[str] = None,
|
||
) -> str:
|
||
"""Study 이름 — 전략·기간·모드 포함."""
|
||
if cli_override and str(cli_override).strip():
|
||
return str(cli_override).strip()
|
||
env_key = f"OPTUNA_{strategy.upper()}_STUDY_NAME"
|
||
from_db = get_env_from_db(env_key, "")
|
||
if from_db and str(from_db).strip() not in ("", "None"):
|
||
return str(from_db).strip()
|
||
legacy = get_env_from_db("OPTUNA_TAIL_STUDY_NAME", "")
|
||
if strategy == "tail" and legacy and str(legacy).strip() not in ("", "None"):
|
||
return str(legacy).strip()
|
||
return f"{strategy}_{mode}_{start}_{end}"
|
||
|
||
|
||
def optuna_run_lock_name(strategy: str) -> str:
|
||
return f"{strategy}_param_search_optuna"
|
||
|
||
|
||
def release_shared_tick_store(ctx: Any, *, log: Optional[logging.Logger] = None) -> None:
|
||
"""
|
||
Optuna ctx.shared_tick_store 해제.
|
||
|
||
주의: ticks_by_code 가 공유메모리 뷰인 경우, unlink 이후 접근하면
|
||
SIGBUS/강제종료(트레이스백 없음) 난다. 최빈(mode_combo) 실측·JSON 저장이
|
||
끝난 뒤에만 호출할 것. optimize() 직후 즉시 unlink 금지.
|
||
"""
|
||
lg = log or logger
|
||
store = getattr(ctx, "shared_tick_store", None)
|
||
if store is None:
|
||
return
|
||
try:
|
||
store.unlink()
|
||
except Exception as exc:
|
||
lg.warning("⚠️ shared_tick_store unlink 실패: %s", exc)
|
||
try:
|
||
ctx.shared_tick_store = None
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
|
||
def announce_optuna_json_path(
|
||
out_path: str,
|
||
*,
|
||
strategy: str = "",
|
||
mode: str = "",
|
||
note: str = "",
|
||
log: Optional[logging.Logger] = None,
|
||
) -> str:
|
||
"""
|
||
결과 JSON 절대경로를 터미널·로그에 눈에 띄게 고지.
|
||
또한 logs/optuna_<strategy>_<mode>_latest.jsonpath 에 기록 (없으면 strategy만).
|
||
note 에 '최종' 이 포함되면 이전장/앞장 브리핑(.briefing.md) 생성.
|
||
"""
|
||
abs_path = os.path.abspath(str(out_path or "").strip())
|
||
lg = log or logger
|
||
tag = note.strip() or "결과 JSON"
|
||
line = f"📁 [{tag}] {abs_path}"
|
||
# logger + print 이중 — nohup 로그·터미널 모두에서 바로 보이게
|
||
lg.info("%s", line)
|
||
print(line, flush=True)
|
||
print(f"OPTUNA_RESULT_JSON={abs_path}", flush=True)
|
||
|
||
try:
|
||
root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||
logs_dir = os.path.join(root, "logs")
|
||
os.makedirs(logs_dir, exist_ok=True)
|
||
s = (strategy or "optuna").strip().lower() or "optuna"
|
||
m = (mode or "run").strip().lower() or "run"
|
||
for name in (
|
||
f"optuna_{s}_{m}_latest.jsonpath",
|
||
f"optuna_{s}_latest.jsonpath",
|
||
"optuna_latest.jsonpath",
|
||
):
|
||
with open(os.path.join(logs_dir, name), "w", encoding="utf-8") as f:
|
||
f.write(abs_path + "\n")
|
||
except OSError as exc:
|
||
lg.warning("⚠️ jsonpath 사이드카 기록 실패: %s", exc)
|
||
|
||
# 최종 JSON 저장 후 브리핑 (이전 장 / 앞으로 장)
|
||
note_l = (note or "").strip()
|
||
if "최종" in note_l and abs_path and os.path.isfile(abs_path):
|
||
try:
|
||
from kis_trader.backtest.optuna_briefing import write_briefing_for_json
|
||
write_briefing_for_json(abs_path, log=lg)
|
||
except Exception as exc:
|
||
lg.warning("⚠️ Optuna 브리핑 실패: %s", exc)
|
||
return abs_path
|
||
|