module Ops class HealthChecks Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true) DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f } DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f } RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes } RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes } SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i } def call findings = [ check_disk_root, check_recordings_size, check_postgres, check_redis, check_mediamtx, check_garage, check_sidekiq_heartbeat, check_sidekiq_dead, check_http_public ] findings.each do |finding| if finding.healthy Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint) else Ops::IncidentRecorder.record( kind: finding.kind, severity: finding.severity, title: finding.title, message: finding.message, metadata: finding.metadata, fingerprint: finding.fingerprint ) end end findings end def summary findings = [ check_disk_root, check_recordings_size, check_postgres, check_redis, check_mediamtx, check_garage, check_sidekiq_heartbeat, check_sidekiq_dead, check_http_public ] status = if findings.any? { |f| !f.healthy && f.severity == "critical" } "down" elsif findings.any? { |f| !f.healthy } "degraded" else "ok" end { status: status, checked_at: Time.current.iso8601, checks: findings.map { |f| finding_to_hash(f) } } end private def check_disk_root disk = Admin::HostMetrics.new.send(:read_disk)[:root] pct = disk[:used_percent].to_f healthy = pct < DISK_WARN_PERCENT.call severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning" Finding.new( kind: "disk_space", severity: severity, healthy: healthy, title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno", message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})", metadata: disk.transform_keys(&:to_s), fingerprint: "disk_space:root" ) end def check_recordings_size path = MatchLiveTv.recordings_local_path disk = Admin::HostMetrics.new.send(:disk_usage_for, path) size = disk[:dir_size_bytes].to_i healthy = size < RECORDINGS_WARN_BYTES.call severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning" Finding.new( kind: "recordings_size", severity: severity, healthy: healthy, title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato", message: "Contenuto #{path}: #{format_bytes(size)}", metadata: { "path" => path, "dir_size_bytes" => size }, fingerprint: "recordings_size:local" ) end def check_postgres ActiveRecord::Base.connection.execute("SELECT 1") ok_finding("service_down:postgres", "PostgreSQL OK") rescue StandardError => e fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message) end def check_redis redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) redis.ping ok_finding("service_down:redis", "Redis OK") rescue StandardError => e fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message) end def check_mediamtx Mediamtx::Client.new.list_paths ok_finding("service_down:mediamtx", "MediaMTX OK") rescue StandardError => e fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message) end def check_garage return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local? require "aws-sdk-s3" client = Aws::S3::Client.new( endpoint: MatchLiveTv.replay_storage_endpoint, region: MatchLiveTv.replay_storage_region, access_key_id: MatchLiveTv.replay_storage_access_key_id, secret_access_key: MatchLiveTv.replay_storage_secret_access_key, force_path_style: MatchLiveTv.replay_storage_force_path_style? ) client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket) ok_finding("garage_storage:ok", "Garage storage OK") rescue StandardError => e fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message) end def check_sidekiq_heartbeat redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i age = last.positive? ? Time.current.to_i - last : nil healthy = age.present? && age < SIDEKIQ_STALE_SECS.call Finding.new( kind: "sidekiq_stale", severity: "critical", healthy: healthy, title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde", message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)", metadata: { "last_heartbeat_age_secs" => age }, fingerprint: "sidekiq_stale:heartbeat" ) end def check_sidekiq_dead dead_count = Sidekiq::DeadSet.new.size healthy = dead_count.zero? severity = dead_count > 5 ? "critical" : "warning" Finding.new( kind: "sidekiq_dead", severity: severity, healthy: healthy, title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue", message: "#{dead_count} job nella dead queue", metadata: { "dead_count" => dead_count }, fingerprint: "sidekiq_dead:count" ) end def check_http_public return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test? url = "#{MatchLiveTv.app_public_url.chomp('/')}/up" response = Faraday.get(url) { |req| req.options.timeout = 10 } healthy = response.status == 200 Finding.new( kind: "http_public", severity: "critical", healthy: healthy, title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde", message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}", metadata: { "url" => url, "status" => response.status }, fingerprint: "http_public:up" ) rescue Faraday::Error => e fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message) end def ok_finding(fingerprint, title) Finding.new( kind: fingerprint.split(":").first, severity: "info", healthy: true, title: title, message: title, metadata: {}, fingerprint: fingerprint ) end def fail_finding(kind, severity, fingerprint, title, message) Finding.new( kind: kind, severity: severity, healthy: false, title: title, message: message, metadata: {}, fingerprint: fingerprint ) end def finding_to_hash(finding) { kind: finding.kind, severity: finding.severity, healthy: finding.healthy, title: finding.title, message: finding.message, metadata: finding.metadata, fingerprint: finding.fingerprint } end def format_bytes(bytes) return "—" if bytes.nil? units = %w[B KB MB GB TB] size = bytes.to_f unit = units.shift while size >= 1024 && units.any? size /= 1024 unit = units.shift end "#{size.round(size >= 10 ? 0 : 1)} #{unit}" end end end