diff --git a/scripts/morning_report.sh b/scripts/morning_report.sh new file mode 100755 index 0000000..da49887 --- /dev/null +++ b/scripts/morning_report.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Morning summary for overnight run logs. + +set -euo pipefail + +LOG_DIR="${LOG_DIR:-data/overnight}" + +if [ ! -d "$LOG_DIR" ]; then + echo "로그 디렉터리가 없습니다: $LOG_DIR" + exit 1 +fi + +latest_run="$(ls -1t "$LOG_DIR"/run_*.log 2>/dev/null | head -n 1 || true)" +latest_watchdog="$(ls -1t "$LOG_DIR"/watchdog_*.log 2>/dev/null | head -n 1 || true)" + +if [ -z "$latest_run" ]; then + echo "run 로그가 없습니다: $LOG_DIR/run_*.log" + exit 1 +fi + +echo "Overnight report" +echo "- run log: $latest_run" +if [ -n "$latest_watchdog" ]; then + echo "- watchdog log: $latest_watchdog" +fi + +start_line="$(head -n 1 "$latest_run" || true)" +end_line="$(tail -n 1 "$latest_run" || true)" + +info_count="$(rg -c '"level": "INFO"' "$latest_run" || true)" +warn_count="$(rg -c '"level": "WARNING"' "$latest_run" || true)" +error_count="$(rg -c '"level": "ERROR"' "$latest_run" || true)" +critical_count="$(rg -c '"level": "CRITICAL"' "$latest_run" || true)" +traceback_count="$(rg -c 'Traceback' "$latest_run" || true)" + +echo "- start: ${start_line:-N/A}" +echo "- end: ${end_line:-N/A}" +echo "- INFO: ${info_count:-0}" +echo "- WARNING: ${warn_count:-0}" +echo "- ERROR: ${error_count:-0}" +echo "- CRITICAL: ${critical_count:-0}" +echo "- Traceback: ${traceback_count:-0}" + +if [ -n "$latest_watchdog" ]; then + watchdog_errors="$(rg -c '\[ERROR\]' "$latest_watchdog" || true)" + echo "- watchdog ERROR: ${watchdog_errors:-0}" + echo "" + echo "최근 watchdog 로그:" + tail -n 5 "$latest_watchdog" || true +fi + +echo "" +echo "최근 앱 로그:" +tail -n 20 "$latest_run" || true diff --git a/scripts/run_overnight.sh b/scripts/run_overnight.sh new file mode 100755 index 0000000..c9c5ae9 --- /dev/null +++ b/scripts/run_overnight.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Start The Ouroboros overnight with logs and watchdog. + +set -euo pipefail + +LOG_DIR="${LOG_DIR:-data/overnight}" +CHECK_INTERVAL="${CHECK_INTERVAL:-30}" +TMUX_AUTO="${TMUX_AUTO:-true}" +TMUX_ATTACH="${TMUX_ATTACH:-true}" +TMUX_SESSION_PREFIX="${TMUX_SESSION_PREFIX:-ouroboros_overnight}" + +if [ -z "${APP_CMD:-}" ]; then + if [ -x ".venv/bin/python" ]; then + PYTHON_BIN=".venv/bin/python" + elif command -v python3 >/dev/null 2>&1; then + PYTHON_BIN="python3" + elif command -v python >/dev/null 2>&1; then + PYTHON_BIN="python" + else + echo ".venv/bin/python 또는 python3/python 실행 파일을 찾을 수 없습니다." + exit 1 + fi + + dashboard_port="${DASHBOARD_PORT:-8080}" + + APP_CMD="DASHBOARD_PORT=$dashboard_port $PYTHON_BIN -m src.main --mode=paper --dashboard" +fi + +mkdir -p "$LOG_DIR" + +timestamp="$(date +"%Y%m%d_%H%M%S")" +RUN_LOG="$LOG_DIR/run_${timestamp}.log" +WATCHDOG_LOG="$LOG_DIR/watchdog_${timestamp}.log" +PID_FILE="$LOG_DIR/app.pid" +WATCHDOG_PID_FILE="$LOG_DIR/watchdog.pid" + +if [ -f "$PID_FILE" ]; then + old_pid="$(cat "$PID_FILE" || true)" + if [ -n "$old_pid" ] && kill -0 "$old_pid" 2>/dev/null; then + echo "앱이 이미 실행 중입니다. pid=$old_pid" + exit 1 + fi +fi + +echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] starting: $APP_CMD" | tee -a "$RUN_LOG" +nohup bash -lc "$APP_CMD" >>"$RUN_LOG" 2>&1 & +app_pid=$! +echo "$app_pid" > "$PID_FILE" + +echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] app pid=$app_pid" | tee -a "$RUN_LOG" + +nohup env PID_FILE="$PID_FILE" LOG_FILE="$WATCHDOG_LOG" CHECK_INTERVAL="$CHECK_INTERVAL" \ + bash scripts/watchdog.sh >/dev/null 2>&1 & +watchdog_pid=$! +echo "$watchdog_pid" > "$WATCHDOG_PID_FILE" + +cat </dev/null 2>&1; then + echo "tmux를 찾지 못해 자동 세션 생성은 건너뜁니다." + exit 0 + fi + + session_name="${TMUX_SESSION_PREFIX}_${timestamp}" + window_name="overnight" + tmux new-session -d -s "$session_name" -n "$window_name" "tail -f '$RUN_LOG'" + tmux split-window -t "${session_name}:${window_name}" -v "tail -f '$WATCHDOG_LOG'" + tmux select-layout -t "${session_name}:${window_name}" even-vertical + + echo "tmux session 생성: $session_name" + echo "수동 접속: tmux attach -t $session_name" + + if [ -z "${TMUX:-}" ] && [ "$TMUX_ATTACH" = "true" ]; then + tmux attach -t "$session_name" + fi +fi diff --git a/scripts/stop_overnight.sh b/scripts/stop_overnight.sh new file mode 100755 index 0000000..f23851e --- /dev/null +++ b/scripts/stop_overnight.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Stop The Ouroboros overnight app/watchdog/tmux session. + +set -euo pipefail + +LOG_DIR="${LOG_DIR:-data/overnight}" +PID_FILE="$LOG_DIR/app.pid" +WATCHDOG_PID_FILE="$LOG_DIR/watchdog.pid" +TMUX_SESSION_PREFIX="${TMUX_SESSION_PREFIX:-ouroboros_overnight}" +KILL_TIMEOUT="${KILL_TIMEOUT:-5}" + +stop_pid() { + local name="$1" + local pid="$2" + + if [ -z "$pid" ]; then + echo "$name PID가 비어 있습니다." + return 1 + fi + + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name 프로세스가 이미 종료됨 (pid=$pid)" + return 0 + fi + + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 "$KILL_TIMEOUT"); do + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name 종료됨 (pid=$pid)" + return 0 + fi + sleep 1 + done + + kill -9 "$pid" 2>/dev/null || true + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name 강제 종료됨 (pid=$pid)" + return 0 + fi + + echo "$name 종료 실패 (pid=$pid)" + return 1 +} + +status=0 + +if [ -f "$WATCHDOG_PID_FILE" ]; then + watchdog_pid="$(cat "$WATCHDOG_PID_FILE" || true)" + stop_pid "watchdog" "$watchdog_pid" || status=1 + rm -f "$WATCHDOG_PID_FILE" +else + echo "watchdog pid 파일 없음: $WATCHDOG_PID_FILE" +fi + +if [ -f "$PID_FILE" ]; then + app_pid="$(cat "$PID_FILE" || true)" + stop_pid "app" "$app_pid" || status=1 + rm -f "$PID_FILE" +else + echo "app pid 파일 없음: $PID_FILE" +fi + +if command -v tmux >/dev/null 2>&1; then + sessions="$(tmux ls 2>/dev/null | awk -F: -v p="$TMUX_SESSION_PREFIX" '$1 ~ "^" p "_" {print $1}')" + if [ -n "$sessions" ]; then + while IFS= read -r s; do + [ -z "$s" ] && continue + tmux kill-session -t "$s" 2>/dev/null || true + echo "tmux 세션 종료: $s" + done <<< "$sessions" + else + echo "종료할 tmux 세션 없음 (prefix=${TMUX_SESSION_PREFIX}_)" + fi +fi + +exit "$status" diff --git a/scripts/watchdog.sh b/scripts/watchdog.sh new file mode 100755 index 0000000..342f3c8 --- /dev/null +++ b/scripts/watchdog.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Simple watchdog for The Ouroboros process. + +set -euo pipefail + +PID_FILE="${PID_FILE:-data/overnight/app.pid}" +LOG_FILE="${LOG_FILE:-data/overnight/watchdog.log}" +CHECK_INTERVAL="${CHECK_INTERVAL:-30}" +STATUS_EVERY="${STATUS_EVERY:-10}" + +mkdir -p "$(dirname "$LOG_FILE")" + +log() { + printf '%s %s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" | tee -a "$LOG_FILE" +} + +if [ ! -f "$PID_FILE" ]; then + log "[ERROR] pid file not found: $PID_FILE" + exit 1 +fi + +PID="$(cat "$PID_FILE")" +if [ -z "$PID" ]; then + log "[ERROR] pid file is empty: $PID_FILE" + exit 1 +fi + +log "[INFO] watchdog started (pid=$PID, interval=${CHECK_INTERVAL}s)" + +count=0 +while true; do + if kill -0 "$PID" 2>/dev/null; then + count=$((count + 1)) + if [ $((count % STATUS_EVERY)) -eq 0 ]; then + log "[INFO] process alive (pid=$PID)" + fi + else + log "[ERROR] process stopped (pid=$PID)" + exit 1 + fi + sleep "$CHECK_INTERVAL" +done