Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
248
backend/app/services/ops/health_checks.rb
Normal file
248
backend/app/services/ops/health_checks.rb
Normal file
@@ -0,0 +1,248 @@
|
||||
module Ops
|
||||
class HealthChecks
|
||||
Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true)
|
||||
|
||||
DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f }
|
||||
DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f }
|
||||
RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes }
|
||||
RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes }
|
||||
SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
|
||||
|
||||
def call
|
||||
findings = [
|
||||
check_disk_root,
|
||||
check_recordings_size,
|
||||
check_postgres,
|
||||
check_redis,
|
||||
check_mediamtx,
|
||||
check_garage,
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_public
|
||||
]
|
||||
|
||||
findings.each do |finding|
|
||||
if finding.healthy
|
||||
Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
|
||||
else
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: finding.kind,
|
||||
severity: finding.severity,
|
||||
title: finding.title,
|
||||
message: finding.message,
|
||||
metadata: finding.metadata,
|
||||
fingerprint: finding.fingerprint
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
findings
|
||||
end
|
||||
|
||||
def summary
|
||||
findings = [
|
||||
check_disk_root,
|
||||
check_recordings_size,
|
||||
check_postgres,
|
||||
check_redis,
|
||||
check_mediamtx,
|
||||
check_garage,
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_public
|
||||
]
|
||||
|
||||
status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
|
||||
"down"
|
||||
elsif findings.any? { |f| !f.healthy }
|
||||
"degraded"
|
||||
else
|
||||
"ok"
|
||||
end
|
||||
|
||||
{
|
||||
status: status,
|
||||
checked_at: Time.current.iso8601,
|
||||
checks: findings.map { |f| finding_to_hash(f) }
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def check_disk_root
|
||||
disk = Admin::HostMetrics.new.send(:read_disk)[:root]
|
||||
pct = disk[:used_percent].to_f
|
||||
healthy = pct < DISK_WARN_PERCENT.call
|
||||
severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "disk_space",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno",
|
||||
message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})",
|
||||
metadata: disk.transform_keys(&:to_s),
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
end
|
||||
|
||||
def check_recordings_size
|
||||
path = MatchLiveTv.recordings_local_path
|
||||
disk = Admin::HostMetrics.new.send(:disk_usage_for, path)
|
||||
size = disk[:dir_size_bytes].to_i
|
||||
healthy = size < RECORDINGS_WARN_BYTES.call
|
||||
severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "recordings_size",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato",
|
||||
message: "Contenuto #{path}: #{format_bytes(size)}",
|
||||
metadata: { "path" => path, "dir_size_bytes" => size },
|
||||
fingerprint: "recordings_size:local"
|
||||
)
|
||||
end
|
||||
|
||||
def check_postgres
|
||||
ActiveRecord::Base.connection.execute("SELECT 1")
|
||||
ok_finding("service_down:postgres", "PostgreSQL OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_redis
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
redis.ping
|
||||
ok_finding("service_down:redis", "Redis OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_mediamtx
|
||||
Mediamtx::Client.new.list_paths
|
||||
ok_finding("service_down:mediamtx", "MediaMTX OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_garage
|
||||
return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local?
|
||||
|
||||
require "aws-sdk-s3"
|
||||
client = Aws::S3::Client.new(
|
||||
endpoint: MatchLiveTv.replay_storage_endpoint,
|
||||
region: MatchLiveTv.replay_storage_region,
|
||||
access_key_id: MatchLiveTv.replay_storage_access_key_id,
|
||||
secret_access_key: MatchLiveTv.replay_storage_secret_access_key,
|
||||
force_path_style: MatchLiveTv.replay_storage_force_path_style?
|
||||
)
|
||||
client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket)
|
||||
ok_finding("garage_storage:ok", "Garage storage OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_sidekiq_heartbeat
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i
|
||||
age = last.positive? ? Time.current.to_i - last : nil
|
||||
healthy = age.present? && age < SIDEKIQ_STALE_SECS.call
|
||||
|
||||
Finding.new(
|
||||
kind: "sidekiq_stale",
|
||||
severity: "critical",
|
||||
healthy: healthy,
|
||||
title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde",
|
||||
message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)",
|
||||
metadata: { "last_heartbeat_age_secs" => age },
|
||||
fingerprint: "sidekiq_stale:heartbeat"
|
||||
)
|
||||
end
|
||||
|
||||
def check_sidekiq_dead
|
||||
dead_count = Sidekiq::DeadSet.new.size
|
||||
healthy = dead_count.zero?
|
||||
severity = dead_count > 5 ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "sidekiq_dead",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue",
|
||||
message: "#{dead_count} job nella dead queue",
|
||||
metadata: { "dead_count" => dead_count },
|
||||
fingerprint: "sidekiq_dead:count"
|
||||
)
|
||||
end
|
||||
|
||||
def check_http_public
|
||||
return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
|
||||
|
||||
url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
|
||||
response = Faraday.get(url) { |req| req.options.timeout = 10 }
|
||||
healthy = response.status == 200
|
||||
|
||||
Finding.new(
|
||||
kind: "http_public",
|
||||
severity: "critical",
|
||||
healthy: healthy,
|
||||
title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
|
||||
message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
|
||||
metadata: { "url" => url, "status" => response.status },
|
||||
fingerprint: "http_public:up"
|
||||
)
|
||||
rescue Faraday::Error => e
|
||||
fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def ok_finding(fingerprint, title)
|
||||
Finding.new(
|
||||
kind: fingerprint.split(":").first,
|
||||
severity: "info",
|
||||
healthy: true,
|
||||
title: title,
|
||||
message: title,
|
||||
metadata: {},
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
end
|
||||
|
||||
def fail_finding(kind, severity, fingerprint, title, message)
|
||||
Finding.new(
|
||||
kind: kind,
|
||||
severity: severity,
|
||||
healthy: false,
|
||||
title: title,
|
||||
message: message,
|
||||
metadata: {},
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
end
|
||||
|
||||
def finding_to_hash(finding)
|
||||
{
|
||||
kind: finding.kind,
|
||||
severity: finding.severity,
|
||||
healthy: finding.healthy,
|
||||
title: finding.title,
|
||||
message: finding.message,
|
||||
metadata: finding.metadata,
|
||||
fingerprint: finding.fingerprint
|
||||
}
|
||||
end
|
||||
|
||||
def format_bytes(bytes)
|
||||
return "—" if bytes.nil?
|
||||
|
||||
units = %w[B KB MB GB TB]
|
||||
size = bytes.to_f
|
||||
unit = units.shift
|
||||
while size >= 1024 && units.any?
|
||||
size /= 1024
|
||||
unit = units.shift
|
||||
end
|
||||
"#{size.round(size >= 10 ? 0 : 1)} #{unit}"
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user