refactor: enhance Optuna backtesting framework, optimize orderbook filtering, and update database management utilities.

scripts/sql/fill_all_config_env_column_comments.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+"""config_* / env_config 전 컬럼 MariaDB COMMENT 채우기 (타입·값 변경 없음).
+
+소스 우선순위:
+  1) 이미 DB에 있는 COMMENT (비어있지 않으면 유지)
+  2) live_config_schema label+hint
+  3) 키 이름 휴리스틱 한글 설명
+  4) 메타 컬럼(id/created_at) 고정문구
+
+실행:
+  cd /home/hoon/kis_bot && python3 scripts/sql/fill_all_config_env_column_comments.py
+  # SQL만: ... --sql-only > scripts/sql/all_config_env_column_comments.generated.sql
+"""
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+
+TABLES = [
+    "config_us_momentum",
+    "config_momentum",
+    "config_short",
+    "config_breakout",
+    "config_updow",
+    "config_scalp",
+    "config_range_break",
+    "config_dbband",
+    "config_dart",
+    "env_config",
+]
+
+# 긴 토큰 우선
+_TOKEN_KO: List[Tuple[str, str]] = [
+    ("STRATEGY", "전략"),
+    ("DAILY_PROFIT", "일일익절"),
+    ("STOP_LOSS", "손절"),
+    ("TAKE_PROFIT", "익절"),
+    ("TRAIL_ARM", "트레일무장"),
+    ("TRAIL", "트레일"),
+    ("SHOULDER_MIN_HIGH", "어깨무장고점"),
+    ("SHOULDER_CUT", "어깨컷"),
+    ("SHOULDER", "어깨"),
+    ("RATCHET_TIERS", "다단래칫"),
+    ("RATCHET", "래칫"),
+    ("ORDERBOOK", "호가"),
+    ("PROGRAM", "프로그램매매"),
+    ("WHIPSAW", "휩쏘"),
+    ("UNIVERSE_SOURCE", "유니버스소스"),
+    ("UNIVERSE", "유니버스"),
+    ("CAND_LIMIT", "후보상한"),
+    ("MAX_STOCKS", "동시보유상한"),
+    ("SLOT_MONEY", "슬롯금액"),
+    ("TOTAL_BUDGET", "총운용한도"),
+    ("MAX_BUY_AMOUNT", "1회최대매수금액"),
+    ("MAX_ORDER_QTY", "주문수량상한"),
+    ("FORCE_BUY", "강제매수테스트"),
+    ("LIVE_MIN_CANDLES", "실매최소봉수"),
+    ("MIN_CANDLES", "최소봉수"),
+    ("MAX_HOLD_BARS", "최대보유봉"),
+    ("MIN_HOLD", "최소보유"),
+    ("HOLD_BARS", "보유봉"),
+    ("MAX_DAILY_CHG", "당일등락상한"),
+    ("MAX_DAILY", "일일매수횟수상한"),
+    ("HIGH_CHASE", "고점추격"),
+    ("FROM_OPEN", "시가대비"),
+    ("PULLBACK", "되돌림"),
+    ("BREAKOUT", "돌파"),
+    ("PATTERN", "패턴"),
+    ("DEFENSE", "방어필터"),
+    ("EMA_FAST", "EMA단기"),
+    ("EMA_SLOW", "EMA장기"),
+    ("EMA", "EMA"),
+    ("RSI_OVERBOUGHT", "RSI과매수"),
+    ("RSI_OVERSOLD", "RSI과매도"),
+    ("RSI_MAX", "RSI상한"),
+    ("RSI_MIN", "RSI하한"),
+    ("RSI", "RSI"),
+    ("VOL_MULT", "거래량배수"),
+    ("VOL_WIN", "거래량윈도우"),
+    ("VOL", "거래량"),
+    ("ATR_SL", "ATR손절"),
+    ("ATR_TP", "ATR익절"),
+    ("ATR", "ATR"),
+    ("EXIT_OB", "수익구간호가매도"),
+    ("STOP_OB", "손절호가"),
+    ("TICK_SELL", "틱매도"),
+    ("TICK_FALLBACK_OHLC", "틱없을때OHLC폴백(위험)"),
+    ("FALLBACK_OHLC", "OHLC폴백"),
+    ("SKIP_HTS_SCAN_DUPES", "HTS스캔중복스킵"),
+    ("SESSION_START", "세션시작"),
+    ("SESSION_END", "세션종료"),
+    ("SELL_TIME_END", "매도세션종료"),
+    ("TIME_START", "매수시작"),
+    ("TIME_END", "매수종료"),
+    ("EOD_ENABLED", "장마감청산ON"),
+    ("EOD_HM", "장마감시각HHMM"),
+    ("EOD", "장마감"),
+    ("DRY_RUN", "모의/페이퍼전용"),
+    ("FEE_RATE", "수수료율"),
+    ("SELL_TAX", "매도세금"),
+    ("FX_FEE", "환전수수료"),
+    ("MIN_PRICE", "최소가격"),
+    ("MAX_PRICE", "최대가격"),
+    ("COOLDOWN", "쿨다운"),
+    ("RELOAD_INTERVAL", "설정리로드주기"),
+    ("LOOP_SLEEP", "루프대기"),
+    ("SCAN_REJECT", "스캔탈락대기"),
+    ("PSAMOUNT", "매수가능수량"),
+    ("BUY_WAIT_LOG", "매수대기로그"),
+    ("APPROVAL", "WS_approval키"),
+    ("REISSUE", "재발급"),
+    ("HOLD_START", "hold시작"),
+    ("HOLD_END", "hold종료"),
+    ("WATCHDOG", "워치독"),
+    ("SILENCE", "틱공백"),
+    ("PING", "프로토콜ping"),
+    ("ORDERBOOK", "호가"),
+    ("ENABLED", "ON/OFF"),
+    ("TARGET", "목표"),
+    ("BUDGET", "한도"),
+    ("HALT", "중단"),
+    ("NOTIFY", "알림"),
+    ("MODE", "모드"),
+    ("TIERS", "다단"),
+    ("DROP", "되돌림/하락"),
+    ("ARM", "무장"),
+    ("PCT", "%"),
+    ("KRW", "원"),
+    ("USD", "USD"),
+    ("SEC", "초"),
+    ("MS", "밀리초"),
+    ("HM", "HHMM"),
+    ("BARS", "봉수"),
+    ("PERIOD", "기간"),
+    ("WINDOW", "윈도우"),
+    ("MULT", "배수"),
+    ("MIN", "최소"),
+    ("MAX", "최대"),
+    ("AVG", "평균"),
+    ("FILTER", "필터"),
+    ("USE_", "사용_"),
+]
+
+_META = {
+    "id": "PK · AUTO_INCREMENT",
+    "created_at": "설정행 생성시각",
+    "updated_at": "마지막 갱신시각",
+}
+
+_TABLE_PREFIX = {
+    "config_us_momentum": "해외모멘텀",
+    "config_momentum": "모멘텀",
+    "config_short": "꼬리잡기",
+    "config_breakout": "돌파",
+    "config_updow": "UPDOWN",
+    "config_scalp": "스캘핑",
+    "config_range_break": "박스권돌파",
+    "config_dbband": "더블BB",
+    "config_dart": "DART",
+    "env_config": "공통env",
+}
+
+
+def _esc_comment(s: str) -> str:
+    """MariaDB COMMENT 문자열 — 작은따옴표 이스케이프, 길이 제한.
+
+    TradeDB/pymysql 은 SQL 의 ``%`` 를 포맷으로 해석 → COMMENT 안 ``%`` 는 ``%%``.
+    """
+    s = re.sub(r"\s+", " ", (s or "").strip())
+    s = s.replace("\\", "\\\\").replace("'", "''")
+    s = s.replace("%", "%%")
+    # MySQL comment 최대 1024; 안전하게 240자 (%% 두 배 전에 자름)
+    if len(s) > 240:
+        s = s[:237] + "..."
+    return s
+
+
+def _heuristic(key: str, table: str) -> str:
+    if key in _META:
+        return _META[key]
+    k = key
+    # strip known strategy prefixes for readability
+    for pref in (
+        "US_MOMENTUM_", "MOMENTUM_", "BREAKOUT_", "SCALP_", "TAIL_", "SHORT_",
+        "UPDOW_", "UPDOWN_", "RANGE_BREAK_", "DBBAND_", "DART_",
+        "KIS_WS_", "KIWOOM_WS_", "LS_WS_", "LS_TOKEN_", "LS_AFR_",
+    ):
+        if k.startswith(pref):
+            k = k[len(pref):]
+            break
+    if k.startswith("STRATEGY_") and k.endswith("_ENABLED"):
+        sid = k[len("STRATEGY_"):-len("_ENABLED")]
+        return f"{sid} 전략 ON/OFF. 다음루프 신규매수중단(보유청산유지). WS구독해제 안함"
+
+    parts: List[str] = []
+    rest = k
+    # greedy token replace
+    while rest:
+        matched = False
+        for tok, ko in _TOKEN_KO:
+            if rest.startswith(tok):
+                parts.append(ko)
+                rest = rest[len(tok):]
+                if rest.startswith("_"):
+                    rest = rest[1:]
+                matched = True
+                break
+        if not matched:
+            # take next underscore chunk
+            if "_" in rest:
+                chunk, rest = rest.split("_", 1)
+            else:
+                chunk, rest = rest, ""
+            parts.append(chunk)
+    body = " ".join(parts) if parts else k
+    scope = _TABLE_PREFIX.get(table, table)
+    # avoid duplicate scope if already in body
+    if scope and scope not in body:
+        return f"{scope}: {body} ({key})"
+    return f"{body} ({key})"
+
+
+def _load_live_labels() -> Dict[str, str]:
+    out: Dict[str, str] = {}
+    try:
+        from kis_trader.web.live_config_schema import build_live_config_groups
+    except Exception as e:
+        print("live_config_schema import 실패:", e)
+        return out
+    for g in build_live_config_groups():
+        for f in g.get("fields") or []:
+            k = f.get("key")
+            if not k:
+                continue
+            lab = str(f.get("label") or "").strip()
+            hint = str(f.get("hint") or "").strip()
+            bits = [b for b in (lab, hint) if b]
+            if bits:
+                out[str(k)] = " · ".join(bits)
+    return out
+
+
+def _describe(key: str, table: str, labels: Dict[str, str], existing: str) -> str:
+    if (existing or "").strip():
+        return existing.strip()
+    if key in labels:
+        return labels[key]
+    return _heuristic(key, table)
+
+
+def _col_type_clause(row: dict) -> str:
+    """SHOW FULL COLUMNS → MODIFY 타입 절 (Null/Default 유지)."""
+    typ = row["Type"]
+    null = "NULL" if row["Null"] == "YES" else "NOT NULL"
+    default = row["Default"]
+    extra = row.get("Extra") or ""
+    parts = [typ, null]
+    if default is None:
+        if row["Null"] == "YES":
+            parts.append("DEFAULT NULL")
+    else:
+        # numeric vs string
+        if re.match(r"^(tinyint|smallint|int|bigint|decimal|float|double|bit)", typ, re.I):
+            parts.append(f"DEFAULT {default}")
+        else:
+            d = str(default).replace("\\", "\\\\").replace("'", "''")
+            parts.append(f"DEFAULT '{d}'")
+    if "auto_increment" in extra.lower():
+        parts.append("AUTO_INCREMENT")
+    return " ".join(parts)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--sql-only", action="store_true")
+    ap.add_argument("--apply", action="store_true", help="DB에 COMMENT 적용")
+    ap.add_argument(
+        "--out",
+        default=str(ROOT / "scripts/sql/all_config_env_column_comments.generated.sql"),
+    )
+    args = ap.parse_args()
+    if not args.sql_only and not args.apply:
+        args.apply = True  # 기본 적용
+
+    labels = _load_live_labels()
+    print(f"live_config labels: {len(labels)}")
+
+    from database import TradeDB
+
+    db = TradeDB()
+    sql_chunks: List[str] = [
+        "-- AUTO-GENERATED: config_*/env_config 전 컬럼 COMMENT (타입·값 불변)",
+        "-- 생성: scripts/sql/fill_all_config_env_column_comments.py",
+        "",
+    ]
+    stats = {"tables": 0, "cols": 0, "kept": 0, "new": 0, "updated": 0}
+
+    for table in TABLES:
+        rows = db.conn.execute(f"SHOW FULL COLUMNS FROM `{table}`").fetchall()
+        stats["tables"] += 1
+        mods: List[str] = []
+        for r in rows:
+            field = r["Field"]
+            stats["cols"] += 1
+            existing = (r.get("Comment") or "").strip()
+            desc = _describe(field, table, labels, existing)
+            if existing:
+                stats["kept"] += 1
+                # 기존 유지 — 그래도 SQL에 명시해 문서화
+            else:
+                stats["new"] += 1
+            if existing == desc and existing:
+                # still emit for generated SQL completeness
+                pass
+            else:
+                if existing and existing != desc:
+                    stats["updated"] += 1  # shouldn't happen (we keep existing)
+            type_clause = _col_type_clause(r)
+            cmt = _esc_comment(desc)
+            mods.append(
+                f"  MODIFY COLUMN `{field}`\n"
+                f"    {type_clause}\n"
+                f"    COMMENT '{cmt}'"
+            )
+        sql_chunks.append(f"-- ── {table} ({len(rows)} cols) ──")
+        sql_chunks.append(f"ALTER TABLE `{table}`")
+        sql_chunks.append(",\n".join(mods))
+        sql_chunks.append(";\n")
+
+        if args.apply:
+            # apply one MODIFY at a time (safer than huge multi)
+            for r in rows:
+                field = r["Field"]
+                existing = (r.get("Comment") or "").strip()
+                desc = _describe(field, table, labels, existing)
+                if existing == desc:
+                    continue
+                type_clause = _col_type_clause(r)
+                cmt = _esc_comment(desc)
+                stmt = (
+                    f"ALTER TABLE `{table}` MODIFY COLUMN `{field}` "
+                    f"{type_clause} COMMENT '{cmt}'"
+                )
+                try:
+                    db.conn.execute(stmt)
+                except Exception as e:
+                    print(f"FAIL {table}.{field}: {e}")
+                    raise
+            print(f"applied {table}")
+
+    out = Path(args.out)
+    out.write_text("\n".join(sql_chunks) + "\n", encoding="utf-8")
+    print(f"wrote {out} ({out.stat().st_size} bytes)")
+    print(
+        f"tables={stats['tables']} cols={stats['cols']} "
+        f"kept_existing={stats['kept']} filled_empty={stats['new']}"
+    )
+
+    # empty remaining?
+    if args.apply:
+        empty = 0
+        for table in TABLES:
+            for r in db.conn.execute(f"SHOW FULL COLUMNS FROM `{table}`").fetchall():
+                if not (r.get("Comment") or "").strip():
+                    empty += 1
+                    print("still empty", table, r["Field"])
+        print("remaining empty comments:", empty)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())