module Ops class HealthChecks Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true) DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f } DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f } RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes } RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes } SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i } def call build_findings.each { |finding| process_finding(finding) } build_findings end def summary findings = build_findings status = if findings.any? { |f| !f.healthy && f.severity == "critical" } "down" elsif findings.any? { |f| !f.healthy } "degraded" else "ok" end { status: status, checked_at: Time.current.iso8601, checks: findings.map { |f| finding_to_hash(f) } } end private def build_findings findings = [ check_disk_root, check_recordings_size, check_postgres, check_redis, check_mediamtx, check_garage, check_sidekiq_heartbeat, check_sidekiq_dead, check_http_rails, check_rails_latency ] findings << check_http_public if public_check_due? findings end def process_finding(finding) if finding.healthy Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint) else Ops::IncidentRecorder.record( kind: finding.kind, severity: finding.severity, title: finding.title, message: finding.message, metadata: finding.metadata, fingerprint: finding.fingerprint ) end end def public_check_due? return false if Rails.env.test? redis = health_redis return true unless redis last = redis.get(public_check_redis_key).to_i due = last.zero? || (Time.current.to_i - last) >= MatchLiveTv.ops_http_public_interval_seconds redis.set(public_check_redis_key, Time.current.to_i) if due due end def public_check_redis_key "ops:health:last_public_check_at" end def health_redis @health_redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) rescue Redis::CannotConnectError nil end def check_disk_root disk = Admin::HostMetrics.new.send(:read_disk)[:root] pct = disk[:used_percent].to_f healthy = pct < DISK_WARN_PERCENT.call severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning" Finding.new( kind: "disk_space", severity: severity, healthy: healthy, title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno", message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})", metadata: disk.transform_keys(&:to_s), fingerprint: "disk_space:root" ) end def check_recordings_size path = MatchLiveTv.recordings_local_path disk = Admin::HostMetrics.new.send(:disk_usage_for, path) size = disk[:dir_size_bytes].to_i healthy = size < RECORDINGS_WARN_BYTES.call severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning" Finding.new( kind: "recordings_size", severity: severity, healthy: healthy, title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato", message: "Contenuto #{path}: #{format_bytes(size)}", metadata: { "path" => path, "dir_size_bytes" => size }, fingerprint: "recordings_size:local" ) end def check_postgres ActiveRecord::Base.connection.execute("SELECT 1") ok_finding("service_down:postgres", "PostgreSQL OK") rescue StandardError => e fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message) end def check_redis redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) redis.ping ok_finding("service_down:redis", "Redis OK") rescue StandardError => e fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message) end def check_mediamtx Mediamtx::Client.new.list_paths ok_finding("service_down:mediamtx", "MediaMTX OK") rescue StandardError => e fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message) end def check_garage return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local? require "aws-sdk-s3" client = Aws::S3::Client.new( endpoint: MatchLiveTv.replay_storage_endpoint, region: MatchLiveTv.replay_storage_region, access_key_id: MatchLiveTv.replay_storage_access_key_id, secret_access_key: MatchLiveTv.replay_storage_secret_access_key, force_path_style: MatchLiveTv.replay_storage_force_path_style? ) client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket) ok_finding("garage_storage:ok", "Garage storage OK") rescue StandardError => e fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message) end def check_sidekiq_heartbeat redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i age = last.positive? ? Time.current.to_i - last : nil healthy = age.present? && age < SIDEKIQ_STALE_SECS.call Finding.new( kind: "sidekiq_stale", severity: "critical", healthy: healthy, title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde", message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)", metadata: { "last_heartbeat_age_secs" => age }, fingerprint: "sidekiq_stale:heartbeat" ) end def check_sidekiq_dead dead_count = Sidekiq::DeadSet.new.size healthy = dead_count.zero? severity = dead_count > 5 ? "critical" : "warning" Finding.new( kind: "sidekiq_dead", severity: severity, healthy: healthy, title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue", message: "#{dead_count} job nella dead queue", metadata: { "dead_count" => dead_count }, fingerprint: "sidekiq_dead:count" ) end def check_http_rails return ok_finding("http_rails:skip", "HTTP Rails check disabilitato in test") if Rails.env.test? url = MatchLiveTv.ops_http_rails_url started = Process.clock_gettime(Process::CLOCK_MONOTONIC) response = Faraday.get(url) { |req| req.options.timeout = 5 } elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round Ops::UpLatencyTracker.record(elapsed_ms) healthy = response.status == 200 Finding.new( kind: "http_rails", severity: "critical", healthy: healthy, title: healthy ? "Rails raggiungibile via edge" : "Rails non risponde via edge", message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}", metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms }, fingerprint: "http_rails:up" ) rescue Faraday::Error => e fail_finding("http_rails", "critical", "http_rails:up", "Rails non raggiungibile via edge", e.message) end def check_rails_latency return ok_finding("rails_latency:skip", "Latenza Rails non valutata in test") if Rails.env.test? min_samples = ENV.fetch("OPS_UP_LATENCY_MIN_SAMPLES", "5").to_i p95 = Ops::UpLatencyTracker.p95 count = Ops::UpLatencyTracker.sample_count if p95.nil? || count < min_samples return ok_finding( "rails_latency:pending", "Latenza /up: campioni insufficienti (#{count}/#{min_samples})" ) end warn_ms = MatchLiveTv.ops_up_latency_warn_ms crit_ms = MatchLiveTv.ops_up_latency_crit_ms healthy = p95 < warn_ms severity = if p95 >= crit_ms "critical" elsif p95 >= warn_ms "warning" else "info" end Finding.new( kind: "rails_latency", severity: severity, healthy: healthy, title: healthy ? "Latenza Rails OK" : "Latenza Rails elevata", message: "p95 GET /up via edge: #{p95}ms (soglia warning #{warn_ms}ms, critical #{crit_ms}ms, n=#{count})", metadata: { "p95_ms" => p95, "samples" => count, "warn_ms" => warn_ms, "crit_ms" => crit_ms }, fingerprint: "rails_latency:p95" ) end def check_http_public return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test? url = "#{MatchLiveTv.app_public_url.chomp('/')}/up" started = Process.clock_gettime(Process::CLOCK_MONOTONIC) response = Faraday.get(url) { |req| req.options.timeout = 10 } elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round healthy = response.status == 200 Finding.new( kind: "http_public", severity: "warning", healthy: healthy, title: healthy ? "Percorso pubblico raggiungibile" : "Percorso pubblico non risponde", message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}", metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms }, fingerprint: "http_public:up" ) rescue Faraday::Error => e fail_finding("http_public", "warning", "http_public:up", "Percorso pubblico non raggiungibile", e.message) end def ok_finding(fingerprint, title) Finding.new( kind: fingerprint.split(":").first, severity: "info", healthy: true, title: title, message: title, metadata: {}, fingerprint: fingerprint ) end def fail_finding(kind, severity, fingerprint, title, message) Finding.new( kind: kind, severity: severity, healthy: false, title: title, message: message, metadata: {}, fingerprint: fingerprint ) end def finding_to_hash(finding) { kind: finding.kind, severity: finding.severity, healthy: finding.healthy, title: finding.title, message: finding.message, metadata: finding.metadata, fingerprint: finding.fingerprint } end def format_bytes(bytes) return "—" if bytes.nil? units = %w[B KB MB GB TB] size = bytes.to_f unit = units.shift while size >= 1024 && units.any? size /= 1024 unit = units.shift end "#{size.round(size >= 10 ? 0 : 1)} #{unit}" end end end