From 22ffdafaccb0bec3f7efed73bc3858878de9b203 Mon Sep 17 00:00:00 2001 From: agentson Date: Tue, 17 Feb 2026 23:24:15 +0900 Subject: [PATCH] chore: add overnight helper scripts - add morning report launcher\n- add overnight stop script\n- add watchdog health monitor script\n\nRefs #137 --- scripts/morning_report.sh | 54 ++++++++++++++++++++++++++++ scripts/stop_overnight.sh | 76 +++++++++++++++++++++++++++++++++++++++ scripts/watchdog.sh | 42 ++++++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100755 scripts/morning_report.sh create mode 100755 scripts/stop_overnight.sh create mode 100755 scripts/watchdog.sh diff --git a/scripts/morning_report.sh b/scripts/morning_report.sh new file mode 100755 index 0000000..da49887 --- /dev/null +++ b/scripts/morning_report.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Morning summary for overnight run logs. + +set -euo pipefail + +LOG_DIR="${LOG_DIR:-data/overnight}" + +if [ ! -d "$LOG_DIR" ]; then + echo "로그 디렉터리가 없습니다: $LOG_DIR" + exit 1 +fi + +latest_run="$(ls -1t "$LOG_DIR"/run_*.log 2>/dev/null | head -n 1 || true)" +latest_watchdog="$(ls -1t "$LOG_DIR"/watchdog_*.log 2>/dev/null | head -n 1 || true)" + +if [ -z "$latest_run" ]; then + echo "run 로그가 없습니다: $LOG_DIR/run_*.log" + exit 1 +fi + +echo "Overnight report" +echo "- run log: $latest_run" +if [ -n "$latest_watchdog" ]; then + echo "- watchdog log: $latest_watchdog" +fi + +start_line="$(head -n 1 "$latest_run" || true)" +end_line="$(tail -n 1 "$latest_run" || true)" + +info_count="$(rg -c '"level": "INFO"' "$latest_run" || true)" +warn_count="$(rg -c '"level": "WARNING"' "$latest_run" || true)" +error_count="$(rg -c '"level": "ERROR"' "$latest_run" || true)" +critical_count="$(rg -c '"level": "CRITICAL"' "$latest_run" || true)" +traceback_count="$(rg -c 'Traceback' "$latest_run" || true)" + +echo "- start: ${start_line:-N/A}" +echo "- end: ${end_line:-N/A}" +echo "- INFO: ${info_count:-0}" +echo "- WARNING: ${warn_count:-0}" +echo "- ERROR: ${error_count:-0}" +echo "- CRITICAL: ${critical_count:-0}" +echo "- Traceback: ${traceback_count:-0}" + +if [ -n "$latest_watchdog" ]; then + watchdog_errors="$(rg -c '\[ERROR\]' "$latest_watchdog" || true)" + echo "- watchdog ERROR: ${watchdog_errors:-0}" + echo "" + echo "최근 watchdog 로그:" + tail -n 5 "$latest_watchdog" || true +fi + +echo "" +echo "최근 앱 로그:" +tail -n 20 "$latest_run" || true diff --git a/scripts/stop_overnight.sh b/scripts/stop_overnight.sh new file mode 100755 index 0000000..f23851e --- /dev/null +++ b/scripts/stop_overnight.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Stop The Ouroboros overnight app/watchdog/tmux session. + +set -euo pipefail + +LOG_DIR="${LOG_DIR:-data/overnight}" +PID_FILE="$LOG_DIR/app.pid" +WATCHDOG_PID_FILE="$LOG_DIR/watchdog.pid" +TMUX_SESSION_PREFIX="${TMUX_SESSION_PREFIX:-ouroboros_overnight}" +KILL_TIMEOUT="${KILL_TIMEOUT:-5}" + +stop_pid() { + local name="$1" + local pid="$2" + + if [ -z "$pid" ]; then + echo "$name PID가 비어 있습니다." + return 1 + fi + + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name 프로세스가 이미 종료됨 (pid=$pid)" + return 0 + fi + + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 "$KILL_TIMEOUT"); do + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name 종료됨 (pid=$pid)" + return 0 + fi + sleep 1 + done + + kill -9 "$pid" 2>/dev/null || true + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name 강제 종료됨 (pid=$pid)" + return 0 + fi + + echo "$name 종료 실패 (pid=$pid)" + return 1 +} + +status=0 + +if [ -f "$WATCHDOG_PID_FILE" ]; then + watchdog_pid="$(cat "$WATCHDOG_PID_FILE" || true)" + stop_pid "watchdog" "$watchdog_pid" || status=1 + rm -f "$WATCHDOG_PID_FILE" +else + echo "watchdog pid 파일 없음: $WATCHDOG_PID_FILE" +fi + +if [ -f "$PID_FILE" ]; then + app_pid="$(cat "$PID_FILE" || true)" + stop_pid "app" "$app_pid" || status=1 + rm -f "$PID_FILE" +else + echo "app pid 파일 없음: $PID_FILE" +fi + +if command -v tmux >/dev/null 2>&1; then + sessions="$(tmux ls 2>/dev/null | awk -F: -v p="$TMUX_SESSION_PREFIX" '$1 ~ "^" p "_" {print $1}')" + if [ -n "$sessions" ]; then + while IFS= read -r s; do + [ -z "$s" ] && continue + tmux kill-session -t "$s" 2>/dev/null || true + echo "tmux 세션 종료: $s" + done <<< "$sessions" + else + echo "종료할 tmux 세션 없음 (prefix=${TMUX_SESSION_PREFIX}_)" + fi +fi + +exit "$status" diff --git a/scripts/watchdog.sh b/scripts/watchdog.sh new file mode 100755 index 0000000..342f3c8 --- /dev/null +++ b/scripts/watchdog.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Simple watchdog for The Ouroboros process. + +set -euo pipefail + +PID_FILE="${PID_FILE:-data/overnight/app.pid}" +LOG_FILE="${LOG_FILE:-data/overnight/watchdog.log}" +CHECK_INTERVAL="${CHECK_INTERVAL:-30}" +STATUS_EVERY="${STATUS_EVERY:-10}" + +mkdir -p "$(dirname "$LOG_FILE")" + +log() { + printf '%s %s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" | tee -a "$LOG_FILE" +} + +if [ ! -f "$PID_FILE" ]; then + log "[ERROR] pid file not found: $PID_FILE" + exit 1 +fi + +PID="$(cat "$PID_FILE")" +if [ -z "$PID" ]; then + log "[ERROR] pid file is empty: $PID_FILE" + exit 1 +fi + +log "[INFO] watchdog started (pid=$PID, interval=${CHECK_INTERVAL}s)" + +count=0 +while true; do + if kill -0 "$PID" 2>/dev/null; then + count=$((count + 1)) + if [ $((count % STATUS_EVERY)) -eq 0 ]; then + log "[INFO] process alive (pid=$PID)" + fi + else + log "[ERROR] process stopped (pid=$PID)" + exit 1 + fi + sleep "$CHECK_INTERVAL" +done