402 lines
13 KiB
Python
Executable File
402 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
kis_error_watch_mm.py — kis_trader_main journalctl 실시간(tail -f) 감시 → Mattermost
|
|
|
|
실매 봇과 분리된 프로세스. journald 만 보고 오류 시 MM 알림.
|
|
- Traceback / FATAL / dead=[...] / 유닛 다운 등
|
|
- 동일·유사 알림은 쿨다운으로 스팸 방지
|
|
- 상태 JSON 즉시 저장(재시작 후에도 쿨다운 유지)
|
|
|
|
실행:
|
|
nohup .venv/bin/python -u scripts/kis_error_watch_mm.py \\
|
|
>> logs/kis_error_watch_mm.log 2>&1 &
|
|
tail -f logs/kis_error_watch_mm.log
|
|
|
|
테스트:
|
|
.venv/bin/python scripts/kis_error_watch_mm.py --test-mm
|
|
|
|
systemd (선택):
|
|
sudo cp deploy/kis_error_watch_mm.service /etc/systemd/system/
|
|
sudo systemctl daemon-reload && sudo systemctl enable --now kis_error_watch_mm
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import List, Optional, Pattern, Tuple
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from kis_trader.utils.env import ( # noqa: E402
|
|
get_env_bool,
|
|
get_env_float,
|
|
get_env_from_db,
|
|
get_env_int,
|
|
)
|
|
from kis_trader.utils.logger import atomic_load_json, atomic_save_json, msg_mm # noqa: E402
|
|
|
|
LOG_PATH = ROOT / "logs" / "kis_error_watch_mm.log"
|
|
STATE_PATH = ROOT / "logs" / "kis_error_watch_mm_state.json"
|
|
LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="[%(asctime)s] %(message)s",
|
|
datefmt="%H:%M:%S",
|
|
handlers=[logging.StreamHandler(sys.stdout)],
|
|
)
|
|
log = logging.getLogger("error_watch")
|
|
|
|
_STOP = False
|
|
|
|
|
|
def _on_signal(signum, _frame) -> None:
|
|
global _STOP
|
|
_STOP = True
|
|
log.info("⏹ signal=%s → 종료 예약", signum)
|
|
|
|
|
|
def _cfg() -> dict:
|
|
"""DB/env 설정 — 하드코딩 수치 금지, get_env_* 만."""
|
|
# Traceback·FATAL·비어있지 않은 dead=·유닛 크래시 시그니처
|
|
default_match = (
|
|
r"(?i)("
|
|
r"Traceback \(most recent call last\)|"
|
|
r"\bCRITICAL\b|\bFATAL\b|MemoryError|SIGBUS|Segmentation fault|"
|
|
r"dead=\[[^\]]|" # dead=[] 제외, dead=['Strat-... 매칭
|
|
r"Main process exited|Failed with result|"
|
|
r"can't open file|"
|
|
r"강제\s*종료|Out of memory|"
|
|
# 운영치명 — 봇 내부 ops_alert 와 이중화 (프로세스 삼킴 대비)
|
|
r"키움 토큰 발급 실패|조건검색 매니저 비활성|"
|
|
r"매니저없음→DB폴백|"
|
|
r"유니버스슬롯.*→0|"
|
|
r"\[MarketGuard\] PANIC|"
|
|
r"매수주문거부|매도 실패|"
|
|
r"LS WS watchdog:.*틱 없음|"
|
|
r"키움 WS LOGIN 실패|"
|
|
r"HTTP 429|KIS rate-limit|"
|
|
r"\[ops_alert\]"
|
|
r")"
|
|
)
|
|
default_ignore = (
|
|
r"(?i)("
|
|
r"numexpr\.utils|"
|
|
r"\[MM 스킵\]|"
|
|
r"MM 발송 실패|"
|
|
r"heartbeat ws=|"
|
|
r"condition 소스 쓰는 전략 없음|"
|
|
r"\[ops_alert\] sent code=|"
|
|
r"ok_mm=True|ok_tg=True|"
|
|
r"\[탈락-|\[TradeDB\] ✅|\[KISWebSocket\] ✅|보유구간 백필"
|
|
r")"
|
|
)
|
|
return {
|
|
"enabled": get_env_bool("ERROR_WATCH_ENABLED", True),
|
|
"unit": str(
|
|
get_env_from_db("ERROR_WATCH_UNIT", "kis_trader_main.service")
|
|
or "kis_trader_main.service"
|
|
).strip(),
|
|
"channel": str(
|
|
get_env_from_db("ERROR_WATCH_MM_CHANNEL", "")
|
|
or get_env_from_db("KIS_SYSTEM_MM_CHANNEL", "default")
|
|
or "default"
|
|
).strip()
|
|
or "default",
|
|
"cooldown_sec": max(30, get_env_int("ERROR_WATCH_COOLDOWN_SEC", 180)),
|
|
"context_lines": max(1, min(20, get_env_int("ERROR_WATCH_CONTEXT_LINES", 5))),
|
|
"traceback_extra": max(0, min(40, get_env_int("ERROR_WATCH_TRACEBACK_EXTRA_LINES", 12))),
|
|
"health_sec": max(15, get_env_int("ERROR_WATCH_HEALTH_CHECK_SEC", 60)),
|
|
"match_re": str(
|
|
get_env_from_db("ERROR_WATCH_MATCH_REGEX", default_match) or default_match
|
|
),
|
|
"ignore_re": str(
|
|
get_env_from_db("ERROR_WATCH_IGNORE_REGEX", default_ignore) or default_ignore
|
|
),
|
|
"jitter": get_env_bool("ERROR_WATCH_MM_JITTER", False),
|
|
}
|
|
|
|
|
|
def _compile_re(pat: str, name: str) -> Optional[Pattern[str]]:
|
|
try:
|
|
return re.compile(pat)
|
|
except re.error as e:
|
|
log.error("❌ regex 컴파일 실패 (%s): %s", name, e)
|
|
return None
|
|
|
|
|
|
def _load_state() -> dict:
|
|
st = atomic_load_json(STATE_PATH, default={})
|
|
if not isinstance(st, dict):
|
|
return {}
|
|
return st
|
|
|
|
|
|
def _save_state(st: dict) -> None:
|
|
atomic_save_json(STATE_PATH, st)
|
|
|
|
|
|
def _fp(text: str) -> str:
|
|
# 시각·PID 제거 후 지문 → 같은 오류 반복 쿨다운
|
|
norm = re.sub(r"\d{2}:\d{2}:\d{2}", "", text)
|
|
norm = re.sub(r"python\[\d+\]", "python[PID]", norm)
|
|
norm = re.sub(r"\s+", " ", norm).strip()[:800]
|
|
return hashlib.sha1(norm.encode("utf-8", errors="ignore")).hexdigest()[:16]
|
|
|
|
|
|
def _can_alert(st: dict, fingerprint: str, cooldown_sec: int) -> bool:
|
|
now = time.time()
|
|
last_ts = float(st.get("last_alert_ts") or 0)
|
|
last_fp = str(st.get("last_fingerprint") or "")
|
|
if fingerprint == last_fp and (now - last_ts) < cooldown_sec:
|
|
return False
|
|
if (now - last_ts) < float(get_env_float("ERROR_WATCH_GLOBAL_MIN_GAP_SEC", 20.0)):
|
|
# 서로 다른 오류라도 최소 간격
|
|
if fingerprint != last_fp and (now - last_ts) < cooldown_sec * 0.15:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _send_alert(title: str, lines: List[str], channel: str, jitter: bool, st: dict, fingerprint: str) -> bool:
|
|
body_lines = [
|
|
f"🚨 **[오류감시] {title}**",
|
|
f"- 시각: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
|
f"- 유닛: `{get_env_from_db('ERROR_WATCH_UNIT', 'kis_trader_main.service')}`",
|
|
"```",
|
|
]
|
|
clipped = "\n".join(lines)[:3500]
|
|
body_lines.append(clipped)
|
|
body_lines.append("```")
|
|
body = "\n".join(body_lines)
|
|
ok = msg_mm(body, channel_alias=channel, jitter=jitter)
|
|
st["last_alert_ts"] = time.time()
|
|
st["last_fingerprint"] = fingerprint
|
|
st["last_title"] = title
|
|
st["alert_count"] = int(st.get("alert_count") or 0) + 1
|
|
_save_state(st)
|
|
log.info("📤 MM %s title=%s fp=%s", "OK" if ok else "FAIL", title, fingerprint)
|
|
return ok
|
|
|
|
|
|
def _unit_active(unit: str) -> Tuple[bool, str]:
|
|
try:
|
|
r = subprocess.run(
|
|
["systemctl", "is-active", unit],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
state = (r.stdout or "").strip() or (r.stderr or "").strip() or "unknown"
|
|
return state == "active", state
|
|
except Exception as e:
|
|
return False, f"check_error:{e}"
|
|
|
|
|
|
def _follow_journal(unit: str) -> subprocess.Popen:
|
|
# -n 0: 과거 덤프 없이 follow만 (기동 직후 과거 Traceback 폭주 방지)
|
|
cmd = [
|
|
"journalctl",
|
|
"-u", unit,
|
|
"-f",
|
|
"-n", "0",
|
|
"--output=short-iso",
|
|
"--no-pager",
|
|
]
|
|
log.info("📡 follow: %s", " ".join(cmd))
|
|
return subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
bufsize=1,
|
|
errors="replace",
|
|
)
|
|
|
|
|
|
def run_watch() -> int:
|
|
cfg = _cfg()
|
|
if not cfg["enabled"]:
|
|
log.warning("ERROR_WATCH_ENABLED=false → 종료")
|
|
return 0
|
|
|
|
match_re = _compile_re(cfg["match_re"], "MATCH")
|
|
ignore_re = _compile_re(cfg["ignore_re"], "IGNORE")
|
|
if match_re is None:
|
|
return 2
|
|
|
|
unit = cfg["unit"]
|
|
channel = cfg["channel"]
|
|
st = _load_state()
|
|
log.info(
|
|
"✅ 감시 시작 unit=%s ch=%s cooldown=%ss health=%ss",
|
|
unit, channel, cfg["cooldown_sec"], cfg["health_sec"],
|
|
)
|
|
|
|
# 기동 알림 (감시자 살아있음 확인)
|
|
if get_env_bool("ERROR_WATCH_STARTUP_NOTIFY", True):
|
|
active, state = _unit_active(unit)
|
|
msg_mm(
|
|
f"👁️ **[오류감시 기동]** `{unit}` → `{state}`"
|
|
f"{' ✅' if active else ' ⚠️ 비활성'}",
|
|
channel_alias=channel,
|
|
jitter=False,
|
|
)
|
|
|
|
proc = _follow_journal(unit)
|
|
buf: List[str] = []
|
|
collecting_tb = False
|
|
tb_left = 0
|
|
last_health = time.time()
|
|
was_active = True
|
|
|
|
assert proc.stdout is not None
|
|
|
|
while not _STOP:
|
|
# health poll
|
|
now = time.time()
|
|
if now - last_health >= cfg["health_sec"]:
|
|
last_health = now
|
|
active, state = _unit_active(unit)
|
|
if not active:
|
|
fp = _fp(f"unit_down:{unit}:{state}")
|
|
if _can_alert(st, fp, cfg["cooldown_sec"]):
|
|
_send_alert(
|
|
f"유닛 비활성 ({state})",
|
|
[f"systemctl is-active {unit} → {state}"],
|
|
channel,
|
|
cfg["jitter"],
|
|
st,
|
|
fp,
|
|
)
|
|
was_active = False
|
|
elif not was_active:
|
|
# 복구 알림
|
|
fp = _fp(f"unit_up:{unit}")
|
|
if _can_alert(st, fp, max(30, cfg["cooldown_sec"] // 3)):
|
|
_send_alert(
|
|
"유닛 복구 (active)",
|
|
[f"systemctl is-active {unit} → active"],
|
|
channel,
|
|
cfg["jitter"],
|
|
st,
|
|
fp,
|
|
)
|
|
was_active = True
|
|
|
|
# journalctl 죽었으면 재기동
|
|
if proc.poll() is not None:
|
|
log.warning("⚠️ journalctl 종료 code=%s → 재기동", proc.returncode)
|
|
proc = _follow_journal(unit)
|
|
assert proc.stdout is not None
|
|
|
|
# non-blocking-ish read with timeout via select
|
|
import select
|
|
|
|
ready, _, _ = select.select([proc.stdout], [], [], 1.0)
|
|
if not ready:
|
|
continue
|
|
line = proc.stdout.readline()
|
|
if line == "":
|
|
# EOF — 재기동
|
|
time.sleep(1.0)
|
|
if proc.poll() is not None:
|
|
proc = _follow_journal(unit)
|
|
assert proc.stdout is not None
|
|
continue
|
|
|
|
line = line.rstrip("\n")
|
|
if not line:
|
|
continue
|
|
|
|
# ignore
|
|
if ignore_re is not None and ignore_re.search(line):
|
|
continue
|
|
|
|
# Traceback 블록 수집
|
|
if "Traceback (most recent call last)" in line:
|
|
collecting_tb = True
|
|
tb_left = cfg["traceback_extra"]
|
|
buf = [line]
|
|
continue
|
|
|
|
if collecting_tb:
|
|
buf.append(line)
|
|
tb_left -= 1
|
|
# 들여쓴 프레임이 끝나고 일반 로그가 오면 종료
|
|
if tb_left <= 0 or (
|
|
len(buf) > 2
|
|
and not line.startswith(" ")
|
|
and not line.startswith("\t")
|
|
and "File \"" not in line
|
|
and not line.lstrip().startswith("File ")
|
|
and "Error" not in line
|
|
and "Exception" not in line
|
|
):
|
|
collecting_tb = False
|
|
block = buf[:]
|
|
buf = []
|
|
fp = _fp("\n".join(block))
|
|
if _can_alert(st, fp, cfg["cooldown_sec"]):
|
|
_send_alert("Traceback", block, channel, cfg["jitter"], st, fp)
|
|
continue
|
|
|
|
if match_re.search(line):
|
|
# 직전 컨텍스트는 journal에 없으므로 히트 라인 + 이후 N줄은 어려움 → 히트만
|
|
ctx = [line]
|
|
fp = _fp(line)
|
|
if _can_alert(st, fp, cfg["cooldown_sec"]):
|
|
title = "로그 오류 매칭"
|
|
if "dead=[" in line:
|
|
title = "전략 dead 감지"
|
|
elif "exited" in line.lower() or "Failed with result" in line:
|
|
title = "프로세스 종료"
|
|
_send_alert(title, ctx, channel, cfg["jitter"], st, fp)
|
|
|
|
try:
|
|
proc.terminate()
|
|
except Exception:
|
|
pass
|
|
log.info("👋 오류감시 종료")
|
|
return 0
|
|
|
|
|
|
def run_test_mm() -> int:
|
|
ch = str(
|
|
get_env_from_db("ERROR_WATCH_MM_CHANNEL", "")
|
|
or get_env_from_db("KIS_SYSTEM_MM_CHANNEL", "default")
|
|
or "default"
|
|
).strip() or "default"
|
|
ok = msg_mm(
|
|
"🧪 **[오류감시 테스트]** kis_error_watch_mm.py --test-mm OK",
|
|
channel_alias=ch,
|
|
jitter=False,
|
|
)
|
|
print(f"test_mm channel={ch} ok={ok}")
|
|
return 0 if ok else 1
|
|
|
|
|
|
def main() -> int:
|
|
signal.signal(signal.SIGINT, _on_signal)
|
|
signal.signal(signal.SIGTERM, _on_signal)
|
|
ap = argparse.ArgumentParser(description="kis_trader journal 오류 → Mattermost")
|
|
ap.add_argument("--test-mm", action="store_true", help="테스트 메시지 1회 발송 후 종료")
|
|
args = ap.parse_args()
|
|
if args.test_mm:
|
|
return run_test_mm()
|
|
return run_watch()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|