MatchLiveTv/backend/app/services/ops/health_checks.rb

module Ops
  class HealthChecks
    Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true)

    DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f }
    DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f }
    RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes }
    RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes }
    SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }

    def call
      build_findings.each { |finding| process_finding(finding) }
      build_findings
    end

    def summary
      findings = build_findings

      status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
                 "down"
               elsif findings.any? { |f| !f.healthy }
                 "degraded"
               else
                 "ok"
               end

      {
        status: status,
        checked_at: Time.current.iso8601,
        checks: findings.map { |f| finding_to_hash(f) }
      }
    end

    private

    def build_findings
      findings = [
        check_disk_root,
        check_recordings_size,
        check_postgres,
        check_redis,
        check_mediamtx,
        check_garage,
        check_sidekiq_heartbeat,
        check_sidekiq_dead,
        check_http_rails,
        check_rails_latency
      ]
      findings << check_http_public if public_check_due?
      findings
    end

    def process_finding(finding)
      if finding.healthy
        Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
      else
        Ops::IncidentRecorder.record(
          kind: finding.kind,
          severity: finding.severity,
          title: finding.title,
          message: finding.message,
          metadata: finding.metadata,
          fingerprint: finding.fingerprint
        )
      end
    end

    def public_check_due?
      return false if Rails.env.test?

      redis = health_redis
      return true unless redis

      last = redis.get(public_check_redis_key).to_i
      due = last.zero? || (Time.current.to_i - last) >= MatchLiveTv.ops_http_public_interval_seconds
      redis.set(public_check_redis_key, Time.current.to_i) if due
      due
    end

    def public_check_redis_key
      "ops:health:last_public_check_at"
    end

    def health_redis
      @health_redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
    rescue Redis::CannotConnectError
      nil
    end

    def check_disk_root
      disk = Admin::HostMetrics.new.send(:read_disk)[:root]
      pct = disk[:used_percent].to_f
      healthy = pct < DISK_WARN_PERCENT.call
      severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning"

      Finding.new(
        kind: "disk_space",
        severity: severity,
        healthy: healthy,
        title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno",
        message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})",
        metadata: disk.transform_keys(&:to_s),
        fingerprint: "disk_space:root"
      )
    end

    def check_recordings_size
      path = MatchLiveTv.recordings_local_path
      disk = Admin::HostMetrics.new.send(:disk_usage_for, path)
      size = disk[:dir_size_bytes].to_i
      healthy = size < RECORDINGS_WARN_BYTES.call
      severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning"

      Finding.new(
        kind: "recordings_size",
        severity: severity,
        healthy: healthy,
        title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato",
        message: "Contenuto #{path}: #{format_bytes(size)}",
        metadata: { "path" => path, "dir_size_bytes" => size },
        fingerprint: "recordings_size:local"
      )
    end

    def check_postgres
      ActiveRecord::Base.connection.execute("SELECT 1")
      ok_finding("service_down:postgres", "PostgreSQL OK")
    rescue StandardError => e
      fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message)
    end

    def check_redis
      redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
      redis.ping
      ok_finding("service_down:redis", "Redis OK")
    rescue StandardError => e
      fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message)
    end

    def check_mediamtx
      Mediamtx::Client.new.list_paths
      ok_finding("service_down:mediamtx", "MediaMTX OK")
    rescue StandardError => e
      fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message)
    end

    def check_garage
      return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local?

      require "aws-sdk-s3"
      client = Aws::S3::Client.new(
        endpoint: MatchLiveTv.replay_storage_endpoint,
        region: MatchLiveTv.replay_storage_region,
        access_key_id: MatchLiveTv.replay_storage_access_key_id,
        secret_access_key: MatchLiveTv.replay_storage_secret_access_key,
        force_path_style: MatchLiveTv.replay_storage_force_path_style?
      )
      client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket)
      ok_finding("garage_storage:ok", "Garage storage OK")
    rescue StandardError => e
      fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
    end

    def check_sidekiq_heartbeat
      redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
      last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i
      age = last.positive? ? Time.current.to_i - last : nil
      healthy = age.present? && age < SIDEKIQ_STALE_SECS.call

      Finding.new(
        kind: "sidekiq_stale",
        severity: "critical",
        healthy: healthy,
        title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde",
        message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)",
        metadata: { "last_heartbeat_age_secs" => age },
        fingerprint: "sidekiq_stale:heartbeat"
      )
    end

    def check_sidekiq_dead
      dead_count = Sidekiq::DeadSet.new.size
      healthy = dead_count.zero?
      severity = dead_count > 5 ? "critical" : "warning"

      Finding.new(
        kind: "sidekiq_dead",
        severity: severity,
        healthy: healthy,
        title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue",
        message: "#{dead_count} job nella dead queue",
        metadata: { "dead_count" => dead_count },
        fingerprint: "sidekiq_dead:count"
      )
    end

    def check_http_rails
      return ok_finding("http_rails:skip", "HTTP Rails check disabilitato in test") if Rails.env.test?

      url = MatchLiveTv.ops_http_rails_url
      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      response = Faraday.get(url) { |req| req.options.timeout = 5 }
      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
      Ops::UpLatencyTracker.record(elapsed_ms)

      healthy = response.status == 200
      Finding.new(
        kind: "http_rails",
        severity: "critical",
        healthy: healthy,
        title: healthy ? "Rails raggiungibile via edge" : "Rails non risponde via edge",
        message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
        metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
        fingerprint: "http_rails:up"
      )
    rescue Faraday::Error => e
      fail_finding("http_rails", "critical", "http_rails:up", "Rails non raggiungibile via edge", e.message)
    end

    def check_rails_latency
      return ok_finding("rails_latency:skip", "Latenza Rails non valutata in test") if Rails.env.test?

      min_samples = ENV.fetch("OPS_UP_LATENCY_MIN_SAMPLES", "5").to_i
      p95 = Ops::UpLatencyTracker.p95
      count = Ops::UpLatencyTracker.sample_count
      if p95.nil? || count < min_samples
        return ok_finding(
          "rails_latency:pending",
          "Latenza /up: campioni insufficienti (#{count}/#{min_samples})"
        )
      end

      warn_ms = MatchLiveTv.ops_up_latency_warn_ms
      crit_ms = MatchLiveTv.ops_up_latency_crit_ms
      healthy = p95 < warn_ms
      severity = if p95 >= crit_ms
                   "critical"
                 elsif p95 >= warn_ms
                   "warning"
                 else
                   "info"
                 end

      Finding.new(
        kind: "rails_latency",
        severity: severity,
        healthy: healthy,
        title: healthy ? "Latenza Rails OK" : "Latenza Rails elevata",
        message: "p95 GET /up via edge: #{p95}ms (soglia warning #{warn_ms}ms, critical #{crit_ms}ms, n=#{count})",
        metadata: { "p95_ms" => p95, "samples" => count, "warn_ms" => warn_ms, "crit_ms" => crit_ms },
        fingerprint: "rails_latency:p95"
      )
    end

    def check_http_public
      return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?

      url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      response = Faraday.get(url) { |req| req.options.timeout = 10 }
      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
      healthy = response.status == 200

      Finding.new(
        kind: "http_public",
        severity: "warning",
        healthy: healthy,
        title: healthy ? "Percorso pubblico raggiungibile" : "Percorso pubblico non risponde",
        message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
        metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
        fingerprint: "http_public:up"
      )
    rescue Faraday::Error => e
      fail_finding("http_public", "warning", "http_public:up", "Percorso pubblico non raggiungibile", e.message)
    end

    def ok_finding(fingerprint, title)
      Finding.new(
        kind: fingerprint.split(":").first,
        severity: "info",
        healthy: true,
        title: title,
        message: title,
        metadata: {},
        fingerprint: fingerprint
      )
    end

    def fail_finding(kind, severity, fingerprint, title, message)
      Finding.new(
        kind: kind,
        severity: severity,
        healthy: false,
        title: title,
        message: message,
        metadata: {},
        fingerprint: fingerprint
      )
    end

    def finding_to_hash(finding)
      {
        kind: finding.kind,
        severity: finding.severity,
        healthy: finding.healthy,
        title: finding.title,
        message: finding.message,
        metadata: finding.metadata,
        fingerprint: finding.fingerprint
      }
    end

    def format_bytes(bytes)
      return "—" if bytes.nil?

      units = %w[B KB MB GB TB]
      size = bytes.to_f
      unit = units.shift
      while size >= 1024 && units.any?
        size /= 1024
        unit = units.shift
      end
      "#{size.round(size >= 10 ? 0 : 1)} #{unit}"
    end
  end
end