Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.

Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep,
dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-12 08:16:47 +02:00
parent ce8939fffb
commit 52cfffb83f
44 changed files with 1396 additions and 24 deletions

View File

@@ -51,6 +51,9 @@ SMTP_AUTH=plain
SMTP_STARTTLS=true
PASSWORD_RESET_EXPIRY_HOURS=2
# Disco video dedicato (recordings MediaMTX + dati Garage). Montare es. 800GB su /media/videos
MATCHLIVETV_VIDEOS_ROOT=/media/videos/matchlivetv
# Replay storage — Garage (obbligatorio in produzione per Premium replay)
# Dopo il primo deploy: bash scripts/setup_garage_production.sh (crea garage.prod.toml + chiavi)
REPLAY_STORAGE_ENDPOINT=http://garage:3900
@@ -59,3 +62,23 @@ REPLAY_STORAGE_REGION=garage
REPLAY_STORAGE_ACCESS_KEY_ID=
REPLAY_STORAGE_SECRET_ACCESS_KEY=
REPLAY_STORAGE_FORCE_PATH_STYLE=true
# Ops monitoring — notifiche push (ntfy) e health check
# App ntfy: https://ntfy.sh — iscriviti al topic e imposta OPS_NTFY_URL
OPS_NTFY_URL=https://ntfy.sh/matchlivetv-ops-YOUR_SECRET_TOPIC
OPS_NTFY_TOKEN=
OPS_ALERT_EMAIL=
OPS_NOTIFY_SEVERITIES=critical
OPS_NOTIFY_COOLDOWN_MINUTES=30
OPS_HEALTH_INTERVAL_SECS=180
OPS_HEALTH_TOKEN=
OPS_DISK_WARN_PERCENT=80
OPS_DISK_CRIT_PERCENT=90
OPS_RECORDINGS_WARN_GB=20
OPS_RECORDINGS_CRIT_GB=50
OPS_SIDEKIQ_STALE_SECS=300
OPS_LOG_SUBSCRIBER=false
# Sentry (opzionale — error tracking produzione)
SENTRY_DSN=
SENTRY_TRACES_SAMPLE_RATE=0.1

View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Scansiona log Docker recenti e invia pattern sospetti a Rails ops:ingest_logs.
set -euo pipefail
ROOT="${MATCHLIVETV_INFRA:-/opt/matchlivetv/infra}"
SINCE="${OPS_LOG_SCAN_SINCE:-10m}"
TMP="$(mktemp)"
trap 'rm -f "$TMP"' EXIT
cd "$ROOT"
docker compose -f docker-compose.prod.yml logs --since "$SINCE" rails sidekiq mediamtx 2>/dev/null > "$TMP" || true
if [[ ! -s "$TMP" ]]; then
exit 0
fi
"${ROOT}/scripts/cron/run_rails_task.sh" ops:ingest_logs["$TMP"]

View File

@@ -4,10 +4,12 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
RUNNER="${ROOT}/scripts/cron/run_rails_task.sh"
SCAN_LOGS="${ROOT}/scripts/cron/scan_ops_logs.sh"
LOG_DIR="${MATCHLIVETV_LOG_DIR:-/opt/matchlivetv/log}"
LOG_FILE="${LOG_DIR}/cron-replay.log"
OPS_LOG_FILE="${LOG_DIR}/cron-ops.log"
chmod +x "$RUNNER"
chmod +x "$RUNNER" "$SCAN_LOGS"
mkdir -p "$LOG_DIR"
MARKER="# matchlivetv-replay-cron"
@@ -15,22 +17,18 @@ CRON_BLOCK="${MARKER}
30 * * * * ${RUNNER} recordings:cleanup_local >> ${LOG_FILE} 2>&1
0 3 * * * ${RUNNER} recordings:purge_expired >> ${LOG_FILE} 2>&1
0 8 * * * ${RUNNER} recordings:expiry_warnings >> ${LOG_FILE} 2>&1
*/10 * * * * ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1
"
existing="$(crontab -l 2>/dev/null || true)"
filtered="$(echo "$existing" | grep -v "$MARKER" | grep -v 'run_rails_task\.sh' || true)"
if echo "$existing" | grep -q "$MARKER"; then
echo "Crontab replay già presente, aggiorno..."
echo "$existing" | awk -v block="$CRON_BLOCK" '
BEGIN { skip=0 }
/# matchlivetv-replay-cron/ { skip=1; next }
skip && /^[^#]/ && $0 !~ /^$/ { skip=0 }
skip { next }
{ print }
END { print block }
' | crontab -
else
(echo "$existing"; echo "$CRON_BLOCK") | crontab -
fi
{
[ -n "$filtered" ] && echo "$filtered"
echo "$CRON_BLOCK"
} | sed '/^$/d' | crontab -
echo "Crontab installato:"
crontab -l | grep -A3 "$MARKER"