Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -51,6 +51,9 @@ SMTP_AUTH=plain
|
||||
SMTP_STARTTLS=true
|
||||
PASSWORD_RESET_EXPIRY_HOURS=2
|
||||
|
||||
# Disco video dedicato (recordings MediaMTX + dati Garage). Montare es. 800GB su /media/videos
|
||||
MATCHLIVETV_VIDEOS_ROOT=/media/videos/matchlivetv
|
||||
|
||||
# Replay storage — Garage (obbligatorio in produzione per Premium replay)
|
||||
# Dopo il primo deploy: bash scripts/setup_garage_production.sh (crea garage.prod.toml + chiavi)
|
||||
REPLAY_STORAGE_ENDPOINT=http://garage:3900
|
||||
@@ -59,3 +62,23 @@ REPLAY_STORAGE_REGION=garage
|
||||
REPLAY_STORAGE_ACCESS_KEY_ID=
|
||||
REPLAY_STORAGE_SECRET_ACCESS_KEY=
|
||||
REPLAY_STORAGE_FORCE_PATH_STYLE=true
|
||||
|
||||
# Ops monitoring — notifiche push (ntfy) e health check
|
||||
# App ntfy: https://ntfy.sh — iscriviti al topic e imposta OPS_NTFY_URL
|
||||
OPS_NTFY_URL=https://ntfy.sh/matchlivetv-ops-YOUR_SECRET_TOPIC
|
||||
OPS_NTFY_TOKEN=
|
||||
OPS_ALERT_EMAIL=
|
||||
OPS_NOTIFY_SEVERITIES=critical
|
||||
OPS_NOTIFY_COOLDOWN_MINUTES=30
|
||||
OPS_HEALTH_INTERVAL_SECS=180
|
||||
OPS_HEALTH_TOKEN=
|
||||
OPS_DISK_WARN_PERCENT=80
|
||||
OPS_DISK_CRIT_PERCENT=90
|
||||
OPS_RECORDINGS_WARN_GB=20
|
||||
OPS_RECORDINGS_CRIT_GB=50
|
||||
OPS_SIDEKIQ_STALE_SECS=300
|
||||
OPS_LOG_SUBSCRIBER=false
|
||||
|
||||
# Sentry (opzionale — error tracking produzione)
|
||||
SENTRY_DSN=
|
||||
SENTRY_TRACES_SAMPLE_RATE=0.1
|
||||
|
||||
18
infra/scripts/cron/scan_ops_logs.sh
Normal file
18
infra/scripts/cron/scan_ops_logs.sh
Normal file
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# Scansiona log Docker recenti e invia pattern sospetti a Rails ops:ingest_logs.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="${MATCHLIVETV_INFRA:-/opt/matchlivetv/infra}"
|
||||
SINCE="${OPS_LOG_SCAN_SINCE:-10m}"
|
||||
TMP="$(mktemp)"
|
||||
trap 'rm -f "$TMP"' EXIT
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
docker compose -f docker-compose.prod.yml logs --since "$SINCE" rails sidekiq mediamtx 2>/dev/null > "$TMP" || true
|
||||
|
||||
if [[ ! -s "$TMP" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
"${ROOT}/scripts/cron/run_rails_task.sh" ops:ingest_logs["$TMP"]
|
||||
@@ -4,10 +4,12 @@ set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
RUNNER="${ROOT}/scripts/cron/run_rails_task.sh"
|
||||
SCAN_LOGS="${ROOT}/scripts/cron/scan_ops_logs.sh"
|
||||
LOG_DIR="${MATCHLIVETV_LOG_DIR:-/opt/matchlivetv/log}"
|
||||
LOG_FILE="${LOG_DIR}/cron-replay.log"
|
||||
OPS_LOG_FILE="${LOG_DIR}/cron-ops.log"
|
||||
|
||||
chmod +x "$RUNNER"
|
||||
chmod +x "$RUNNER" "$SCAN_LOGS"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
MARKER="# matchlivetv-replay-cron"
|
||||
@@ -15,22 +17,18 @@ CRON_BLOCK="${MARKER}
|
||||
30 * * * * ${RUNNER} recordings:cleanup_local >> ${LOG_FILE} 2>&1
|
||||
0 3 * * * ${RUNNER} recordings:purge_expired >> ${LOG_FILE} 2>&1
|
||||
0 8 * * * ${RUNNER} recordings:expiry_warnings >> ${LOG_FILE} 2>&1
|
||||
*/10 * * * * ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1
|
||||
"
|
||||
|
||||
existing="$(crontab -l 2>/dev/null || true)"
|
||||
filtered="$(echo "$existing" | grep -v "$MARKER" | grep -v 'run_rails_task\.sh' || true)"
|
||||
if echo "$existing" | grep -q "$MARKER"; then
|
||||
echo "Crontab replay già presente, aggiorno..."
|
||||
echo "$existing" | awk -v block="$CRON_BLOCK" '
|
||||
BEGIN { skip=0 }
|
||||
/# matchlivetv-replay-cron/ { skip=1; next }
|
||||
skip && /^[^#]/ && $0 !~ /^$/ { skip=0 }
|
||||
skip { next }
|
||||
{ print }
|
||||
END { print block }
|
||||
' | crontab -
|
||||
else
|
||||
(echo "$existing"; echo "$CRON_BLOCK") | crontab -
|
||||
fi
|
||||
{
|
||||
[ -n "$filtered" ] && echo "$filtered"
|
||||
echo "$CRON_BLOCK"
|
||||
} | sed '/^$/d' | crontab -
|
||||
|
||||
echo "Crontab installato:"
|
||||
crontab -l | grep -A3 "$MARKER"
|
||||
|
||||
Reference in New Issue
Block a user