Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.

Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep,
dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/services/admin/host_metrics.rb

@@ -84,26 +84,57 @@ module Admin
     def disk_usage_for(path)
       return unavailable_disk unless path.present? && (Dir.exist?(path) || File.exist?(path))
 
+      usage = disk_usage_via_statvfs(path) || disk_usage_via_df(path)
+      return unavailable_disk unless usage
+
+      usage.merge(dir_size_bytes: recordings_dir_size(path))
+    rescue StandardError
+      unavailable_disk
+    end
+
+    def disk_usage_via_statvfs(path)
+      return nil unless File.respond_to?(:statvfs)
+
       stat = File.statvfs(path)
       total = stat.blocks * stat.frsize
       free = stat.bavail * stat.frsize
       used = total - free
+      build_disk_usage(path, total: total, used: used, free: free)
+    rescue StandardError
+      nil
+    end
+
+    def disk_usage_via_df(path)
+      output = shell_out("df -B1 --output=size,used,avail,pcent,target #{Shellwords.escape(path)} 2>/dev/null").strip
+      line = output.lines.map(&:strip).reject(&:blank?).last
+      return nil if line.blank? || line.start_with?("Filesystem")
+
+      parts = line.split
+      return nil if parts.size < 5
+
+      total = parts[0].to_i
+      used = parts[1].to_i
+      free = parts[2].to_i
+      pct = parts[3].delete("%").to_f
+      build_disk_usage(path, total: total, used: used, free: free, used_percent: pct)
+    rescue StandardError
+      nil
+    end
+
+    def build_disk_usage(path, total:, used:, free:, used_percent: nil)
       {
         path: path,
         total_bytes: total,
         used_bytes: used,
         free_bytes: free,
-        used_percent: total.positive? ? ((used.to_f / total) * 100).round(1) : 0.0,
-        dir_size_bytes: recordings_dir_size(path)
+        used_percent: used_percent || (total.positive? ? ((used.to_f / total) * 100).round(1) : 0.0)
       }
-    rescue StandardError
-      unavailable_disk
     end
 
     def recordings_dir_size(path)
       return nil unless path == RECORDINGS_PATH && Dir.exist?(path)
 
-      `du -sb #{Shellwords.escape(path)} 2>/dev/null`.split.first.to_i
+      shell_out("du -sb #{Shellwords.escape(path)} 2>/dev/null").split.first.to_i
     rescue StandardError
       nil
     end
@@ -112,6 +143,10 @@ module Admin
       { path: nil, total_bytes: nil, used_bytes: nil, free_bytes: nil, used_percent: nil, dir_size_bytes: nil }
     end
 
+    def shell_out(cmd)
+      `#{cmd}`
+    end
+
     def read_network
       rx, tx = parse_net_bytes
       prev = MetricsStore.net_totals

backend/app/services/ops/health_checks.rb

@@ -0,0 +1,248 @@
+module Ops
+  class HealthChecks
+    Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true)
+
+    DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f }
+    DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f }
+    RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes }
+    RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes }
+    SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
+
+    def call
+      findings = [
+        check_disk_root,
+        check_recordings_size,
+        check_postgres,
+        check_redis,
+        check_mediamtx,
+        check_garage,
+        check_sidekiq_heartbeat,
+        check_sidekiq_dead,
+        check_http_public
+      ]
+
+      findings.each do |finding|
+        if finding.healthy
+          Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
+        else
+          Ops::IncidentRecorder.record(
+            kind: finding.kind,
+            severity: finding.severity,
+            title: finding.title,
+            message: finding.message,
+            metadata: finding.metadata,
+            fingerprint: finding.fingerprint
+          )
+        end
+      end
+
+      findings
+    end
+
+    def summary
+      findings = [
+        check_disk_root,
+        check_recordings_size,
+        check_postgres,
+        check_redis,
+        check_mediamtx,
+        check_garage,
+        check_sidekiq_heartbeat,
+        check_sidekiq_dead,
+        check_http_public
+      ]
+
+      status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
+                 "down"
+               elsif findings.any? { |f| !f.healthy }
+                 "degraded"
+               else
+                 "ok"
+               end
+
+      {
+        status: status,
+        checked_at: Time.current.iso8601,
+        checks: findings.map { |f| finding_to_hash(f) }
+      }
+    end
+
+    private
+
+    def check_disk_root
+      disk = Admin::HostMetrics.new.send(:read_disk)[:root]
+      pct = disk[:used_percent].to_f
+      healthy = pct < DISK_WARN_PERCENT.call
+      severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning"
+
+      Finding.new(
+        kind: "disk_space",
+        severity: severity,
+        healthy: healthy,
+        title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno",
+        message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})",
+        metadata: disk.transform_keys(&:to_s),
+        fingerprint: "disk_space:root"
+      )
+    end
+
+    def check_recordings_size
+      path = MatchLiveTv.recordings_local_path
+      disk = Admin::HostMetrics.new.send(:disk_usage_for, path)
+      size = disk[:dir_size_bytes].to_i
+      healthy = size < RECORDINGS_WARN_BYTES.call
+      severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning"
+
+      Finding.new(
+        kind: "recordings_size",
+        severity: severity,
+        healthy: healthy,
+        title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato",
+        message: "Contenuto #{path}: #{format_bytes(size)}",
+        metadata: { "path" => path, "dir_size_bytes" => size },
+        fingerprint: "recordings_size:local"
+      )
+    end
+
+    def check_postgres
+      ActiveRecord::Base.connection.execute("SELECT 1")
+      ok_finding("service_down:postgres", "PostgreSQL OK")
+    rescue StandardError => e
+      fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message)
+    end
+
+    def check_redis
+      redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+      redis.ping
+      ok_finding("service_down:redis", "Redis OK")
+    rescue StandardError => e
+      fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message)
+    end
+
+    def check_mediamtx
+      Mediamtx::Client.new.list_paths
+      ok_finding("service_down:mediamtx", "MediaMTX OK")
+    rescue StandardError => e
+      fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message)
+    end
+
+    def check_garage
+      return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local?
+
+      require "aws-sdk-s3"
+      client = Aws::S3::Client.new(
+        endpoint: MatchLiveTv.replay_storage_endpoint,
+        region: MatchLiveTv.replay_storage_region,
+        access_key_id: MatchLiveTv.replay_storage_access_key_id,
+        secret_access_key: MatchLiveTv.replay_storage_secret_access_key,
+        force_path_style: MatchLiveTv.replay_storage_force_path_style?
+      )
+      client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket)
+      ok_finding("garage_storage:ok", "Garage storage OK")
+    rescue StandardError => e
+      fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
+    end
+
+    def check_sidekiq_heartbeat
+      redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+      last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i
+      age = last.positive? ? Time.current.to_i - last : nil
+      healthy = age.present? && age < SIDEKIQ_STALE_SECS.call
+
+      Finding.new(
+        kind: "sidekiq_stale",
+        severity: "critical",
+        healthy: healthy,
+        title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde",
+        message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)",
+        metadata: { "last_heartbeat_age_secs" => age },
+        fingerprint: "sidekiq_stale:heartbeat"
+      )
+    end
+
+    def check_sidekiq_dead
+      dead_count = Sidekiq::DeadSet.new.size
+      healthy = dead_count.zero?
+      severity = dead_count > 5 ? "critical" : "warning"
+
+      Finding.new(
+        kind: "sidekiq_dead",
+        severity: severity,
+        healthy: healthy,
+        title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue",
+        message: "#{dead_count} job nella dead queue",
+        metadata: { "dead_count" => dead_count },
+        fingerprint: "sidekiq_dead:count"
+      )
+    end
+
+    def check_http_public
+      return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
+
+      url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
+      response = Faraday.get(url) { |req| req.options.timeout = 10 }
+      healthy = response.status == 200
+
+      Finding.new(
+        kind: "http_public",
+        severity: "critical",
+        healthy: healthy,
+        title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
+        message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
+        metadata: { "url" => url, "status" => response.status },
+        fingerprint: "http_public:up"
+      )
+    rescue Faraday::Error => e
+      fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
+    end
+
+    def ok_finding(fingerprint, title)
+      Finding.new(
+        kind: fingerprint.split(":").first,
+        severity: "info",
+        healthy: true,
+        title: title,
+        message: title,
+        metadata: {},
+        fingerprint: fingerprint
+      )
+    end
+
+    def fail_finding(kind, severity, fingerprint, title, message)
+      Finding.new(
+        kind: kind,
+        severity: severity,
+        healthy: false,
+        title: title,
+        message: message,
+        metadata: {},
+        fingerprint: fingerprint
+      )
+    end
+
+    def finding_to_hash(finding)
+      {
+        kind: finding.kind,
+        severity: finding.severity,
+        healthy: finding.healthy,
+        title: finding.title,
+        message: finding.message,
+        metadata: finding.metadata,
+        fingerprint: finding.fingerprint
+      }
+    end
+
+    def format_bytes(bytes)
+      return "—" if bytes.nil?
+
+      units = %w[B KB MB GB TB]
+      size = bytes.to_f
+      unit = units.shift
+      while size >= 1024 && units.any?
+        size /= 1024
+        unit = units.shift
+      end
+      "#{size.round(size >= 10 ? 0 : 1)} #{unit}"
+    end
+  end
+end

backend/app/services/ops/incident_recorder.rb

@@ -0,0 +1,67 @@
+module Ops
+  class IncidentRecorder
+    COOLDOWN = -> { ENV.fetch("OPS_NOTIFY_COOLDOWN_MINUTES", "30").to_i.minutes }
+
+    class << self
+      def record(finding)
+        new.record(finding)
+      end
+
+      def resolve(fingerprint:)
+        new.resolve(fingerprint: fingerprint)
+      end
+    end
+
+    def record(finding)
+      fingerprint = finding[:fingerprint] || fingerprint_for(finding)
+      incident = Ops::Incident.open.find_by(fingerprint: fingerprint)
+
+      if incident
+        incident.occurrence_count += 1
+        incident.last_seen_at = Time.current
+        incident.title = finding[:title] if finding[:title].present?
+        incident.message = finding[:message] if finding[:message].present?
+        incident.metadata = (incident.metadata || {}).merge(finding[:metadata] || {})
+        incident.severity = finding[:severity] if finding[:severity].present?
+        incident.status = "open" if incident.status == "acknowledged"
+      else
+        now = Time.current
+        incident = Ops::Incident.new(
+          kind: finding[:kind],
+          severity: finding[:severity],
+          status: "open",
+          title: finding[:title],
+          message: finding[:message],
+          metadata: finding[:metadata] || {},
+          fingerprint: fingerprint,
+          occurrence_count: 1,
+          first_seen_at: now,
+          last_seen_at: now
+        )
+      end
+
+      incident.save!
+      maybe_notify!(incident)
+      incident
+    end
+
+    def resolve(fingerprint:)
+      Ops::Incident.open.where(fingerprint: fingerprint).find_each(&:resolve!)
+    end
+
+    private
+
+    def fingerprint_for(finding)
+      Digest::SHA256.hexdigest([finding[:kind], finding[:title]].join("|"))
+    end
+
+    def maybe_notify!(incident)
+      return if incident.muted?
+      return unless Ops::Notifier.notify_severity?(incident.severity)
+      return if incident.notified_at.present? && incident.notified_at > COOLDOWN.call.ago
+
+      Ops::Notifier.new.notify(incident)
+      incident.update_column(:notified_at, Time.current)
+    end
+  end
+end

backend/app/services/ops/log_pattern_matcher.rb

@@ -0,0 +1,37 @@
+module Ops
+  class LogPatternMatcher
+    Match = Struct.new(:name, :severity, :line, keyword_init: true)
+
+    def initialize(patterns: nil)
+      @patterns = patterns || load_patterns
+    end
+
+    def scan(text)
+      return [] if text.blank?
+
+      matches = []
+      text.each_line do |line|
+        @patterns.each do |pattern|
+          next unless line.match?(pattern[:regex])
+
+          matches << Match.new(name: pattern[:name], severity: pattern[:severity], line: line.strip)
+        end
+      end
+      matches
+    end
+
+    private
+
+    def load_patterns
+      path = Rails.root.join("config/ops_log_patterns.yml")
+      raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: true) || {}
+      Array(raw["patterns"]).map do |entry|
+        {
+          name: entry["name"].to_s,
+          severity: entry["severity"].to_s,
+          regex: Regexp.new(entry["regex"], Regexp::IGNORECASE)
+        }
+      end
+    end
+  end
+end

backend/app/services/ops/log_scanner.rb

@@ -0,0 +1,26 @@
+module Ops
+  class LogScanner
+    def initialize(matcher: LogPatternMatcher.new)
+      @matcher = matcher
+    end
+
+    def ingest(text)
+      matches = @matcher.scan(text)
+      matches.each { |match| record_match(match) }
+      matches.size
+    end
+
+    private
+
+    def record_match(match)
+      Ops::IncidentRecorder.record(
+        kind: "log_pattern",
+        severity: match.severity,
+        title: "Log: #{match.name}",
+        message: match.line.truncate(500),
+        metadata: { "pattern" => match.name, "sample" => match.line },
+        fingerprint: "log_pattern:#{match.name}"
+      )
+    end
+  end
+end

backend/app/services/ops/notifier.rb

@@ -0,0 +1,65 @@
+module Ops
+  class Notifier
+    class Error < StandardError; end
+
+    def self.notify_severity?(severity)
+      levels = ENV.fetch("OPS_NOTIFY_SEVERITIES", "critical").split(",").map(&:strip)
+      levels.include?(severity.to_s)
+    end
+
+    def notify(incident)
+      notify_ntfy(incident) if ntfy_url.present?
+      notify_email(incident) if alert_email.present?
+    end
+
+    private
+
+    def notify_ntfy(incident)
+      priority = incident.severity == "critical" ? "5" : "3"
+      tags = incident.severity == "critical" ? "rotating_light" : "warning"
+
+      conn = Faraday.new do |f|
+        f.adapter Faraday.default_adapter
+        f.options.timeout = 10
+        f.options.open_timeout = 5
+      end
+
+      headers = {
+        "Title" => "[Match Live TV] #{incident.title}",
+        "Priority" => priority,
+        "Tags" => tags
+      }
+      headers["Authorization"] = "Bearer #{ntfy_token}" if ntfy_token.present?
+
+      body = incident.message.to_s
+      body += "\n\n(#{incident.occurrence_count} occorrenze)" if incident.occurrence_count > 1
+
+      response = conn.post(ntfy_url, body, headers)
+      return if response.success?
+
+      Rails.logger.warn("[Ops::Notifier] ntfy failed: #{response.status} #{response.body}")
+    rescue Faraday::Error => e
+      Rails.logger.warn("[Ops::Notifier] ntfy error: #{e.message}")
+    end
+
+    def notify_email(incident)
+      return unless defined?(ActionMailer)
+
+      Ops::AlertMailer.incident(incident).deliver_later
+    rescue StandardError => e
+      Rails.logger.warn("[Ops::Notifier] email error: #{e.message}")
+    end
+
+    def ntfy_url
+      ENV["OPS_NTFY_URL"].presence
+    end
+
+    def ntfy_token
+      ENV["OPS_NTFY_TOKEN"].presence
+    end
+
+    def alert_email
+      ENV["OPS_ALERT_EMAIL"].presence
+    end
+  end
+end