HLS su edge e monitoraggio ops con latenza p95.

Evita di saturare Puma in diretta, separa i check Rails/pubblico e traccia la latenza /up per alert più affidabili.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/models/ops/incident.rb

@@ -3,8 +3,8 @@ module Ops
     self.table_name = "ops_incidents"
 
     KINDS = %w[
-      disk_space recordings_size service_down http_public sidekiq_stale sidekiq_dead
-      log_pattern garage_storage
+      disk_space recordings_size service_down http_public http_rails rails_latency
+      sidekiq_stale sidekiq_dead log_pattern garage_storage
     ].freeze
     SEVERITIES = %w[critical warning info].freeze
     STATUSES = %w[open acknowledged resolved].freeze

backend/app/services/ops/health_checks.rb

@@ -9,48 +9,12 @@ module Ops
     SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
 
     def call
-      findings = [
-        check_disk_root,
-        check_recordings_size,
-        check_postgres,
-        check_redis,
-        check_mediamtx,
-        check_garage,
-        check_sidekiq_heartbeat,
-        check_sidekiq_dead,
-        check_http_public
-      ]
-
-      findings.each do |finding|
-        if finding.healthy
-          Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
-        else
-          Ops::IncidentRecorder.record(
-            kind: finding.kind,
-            severity: finding.severity,
-            title: finding.title,
-            message: finding.message,
-            metadata: finding.metadata,
-            fingerprint: finding.fingerprint
-          )
-        end
-      end
-
-      findings
+      build_findings.each { |finding| process_finding(finding) }
+      build_findings
     end
 
     def summary
-      findings = [
-        check_disk_root,
-        check_recordings_size,
-        check_postgres,
-        check_redis,
-        check_mediamtx,
-        check_garage,
-        check_sidekiq_heartbeat,
-        check_sidekiq_dead,
-        check_http_public
-      ]
+      findings = build_findings
 
       status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
                  "down"
@@ -69,6 +33,60 @@ module Ops
 
     private
 
+    def build_findings
+      findings = [
+        check_disk_root,
+        check_recordings_size,
+        check_postgres,
+        check_redis,
+        check_mediamtx,
+        check_garage,
+        check_sidekiq_heartbeat,
+        check_sidekiq_dead,
+        check_http_rails,
+        check_rails_latency
+      ]
+      findings << check_http_public if public_check_due?
+      findings
+    end
+
+    def process_finding(finding)
+      if finding.healthy
+        Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
+      else
+        Ops::IncidentRecorder.record(
+          kind: finding.kind,
+          severity: finding.severity,
+          title: finding.title,
+          message: finding.message,
+          metadata: finding.metadata,
+          fingerprint: finding.fingerprint
+        )
+      end
+    end
+
+    def public_check_due?
+      return false if Rails.env.test?
+
+      redis = health_redis
+      return true unless redis
+
+      last = redis.get(public_check_redis_key).to_i
+      due = last.zero? || (Time.current.to_i - last) >= MatchLiveTv.ops_http_public_interval_seconds
+      redis.set(public_check_redis_key, Time.current.to_i) if due
+      due
+    end
+
+    def public_check_redis_key
+      "ops:health:last_public_check_at"
+    end
+
+    def health_redis
+      @health_redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+    rescue Redis::CannotConnectError
+      nil
+    end
+
     def check_disk_root
       disk = Admin::HostMetrics.new.send(:read_disk)[:root]
       pct = disk[:used_percent].to_f
@@ -176,24 +194,84 @@ module Ops
       )
     end
 
+    def check_http_rails
+      return ok_finding("http_rails:skip", "HTTP Rails check disabilitato in test") if Rails.env.test?
+
+      url = MatchLiveTv.ops_http_rails_url
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      response = Faraday.get(url) { |req| req.options.timeout = 5 }
+      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
+      Ops::UpLatencyTracker.record(elapsed_ms)
+
+      healthy = response.status == 200
+      Finding.new(
+        kind: "http_rails",
+        severity: "critical",
+        healthy: healthy,
+        title: healthy ? "Rails raggiungibile via edge" : "Rails non risponde via edge",
+        message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
+        metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
+        fingerprint: "http_rails:up"
+      )
+    rescue Faraday::Error => e
+      fail_finding("http_rails", "critical", "http_rails:up", "Rails non raggiungibile via edge", e.message)
+    end
+
+    def check_rails_latency
+      return ok_finding("rails_latency:skip", "Latenza Rails non valutata in test") if Rails.env.test?
+
+      min_samples = ENV.fetch("OPS_UP_LATENCY_MIN_SAMPLES", "5").to_i
+      p95 = Ops::UpLatencyTracker.p95
+      count = Ops::UpLatencyTracker.sample_count
+      if p95.nil? || count < min_samples
+        return ok_finding(
+          "rails_latency:pending",
+          "Latenza /up: campioni insufficienti (#{count}/#{min_samples})"
+        )
+      end
+
+      warn_ms = MatchLiveTv.ops_up_latency_warn_ms
+      crit_ms = MatchLiveTv.ops_up_latency_crit_ms
+      healthy = p95 < warn_ms
+      severity = if p95 >= crit_ms
+                   "critical"
+                 elsif p95 >= warn_ms
+                   "warning"
+                 else
+                   "info"
+                 end
+
+      Finding.new(
+        kind: "rails_latency",
+        severity: severity,
+        healthy: healthy,
+        title: healthy ? "Latenza Rails OK" : "Latenza Rails elevata",
+        message: "p95 GET /up via edge: #{p95}ms (soglia warning #{warn_ms}ms, critical #{crit_ms}ms, n=#{count})",
+        metadata: { "p95_ms" => p95, "samples" => count, "warn_ms" => warn_ms, "crit_ms" => crit_ms },
+        fingerprint: "rails_latency:p95"
+      )
+    end
+
     def check_http_public
       return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
 
       url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       response = Faraday.get(url) { |req| req.options.timeout = 10 }
+      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
       healthy = response.status == 200
 
       Finding.new(
         kind: "http_public",
-        severity: "critical",
+        severity: "warning",
         healthy: healthy,
-        title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
-        message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
-        metadata: { "url" => url, "status" => response.status },
+        title: healthy ? "Percorso pubblico raggiungibile" : "Percorso pubblico non risponde",
+        message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
+        metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
         fingerprint: "http_public:up"
       )
     rescue Faraday::Error => e
-      fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
+      fail_finding("http_public", "warning", "http_public:up", "Percorso pubblico non raggiungibile", e.message)
     end
 
     def ok_finding(fingerprint, title)

backend/app/services/ops/up_latency_tracker.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Ops
+  # Campioni latenza GET /up (edge → rails) per calcolo p95.
+  class UpLatencyTracker
+    REDIS_KEY = "ops:up_latency_ms"
+    MAX_SAMPLES = -> { ENV.fetch("OPS_UP_LATENCY_SAMPLES", "60").to_i }
+
+    class << self
+      def record(latency_ms)
+        return unless redis
+
+        redis.pipelined do |pipe|
+          pipe.lpush(REDIS_KEY, latency_ms.to_i)
+          pipe.ltrim(REDIS_KEY, 0, MAX_SAMPLES.call - 1)
+        end
+      end
+
+      def samples
+        return [] unless redis
+
+        redis.lrange(REDIS_KEY, 0, -1).map(&:to_i)
+      end
+
+      def sample_count
+        samples.size
+      end
+
+      def p95
+        sorted = samples.sort
+        return nil if sorted.empty?
+
+        index = [(sorted.size * 0.95).ceil - 1, 0].max
+        sorted[index]
+      end
+
+      def redis
+        @redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+      rescue Redis::CannotConnectError
+        nil
+      end
+    end
+  end
+end

backend/app/services/streams/youtube_relay.rb

@@ -153,8 +153,8 @@ module Streams
         end
 
         path = session.mediamtx_path_name
-        rails = ENV.fetch("RAILS_INTERNAL_URL", "http://rails:3000")
-        [:hls, "#{rails.chomp('/')}/hls/#{path}/index.m3u8"]
+        hls = ENV.fetch("MEDIAMTX_HLS_URL", "http://mediamtx:8888").chomp("/")
+        [:hls, "#{hls}/#{path}/index.m3u8"]
       end
 
       def intake_available?(session)

backend/config/initializers/match_live_tv.rb

@@ -151,6 +151,22 @@ module MatchLiveTv
       ENV.fetch("REPLAY_MEDIA_REDIRECT", "true") == "true"
     end
 
+    def ops_http_rails_url
+      ENV.fetch("OPS_HTTP_RAILS_URL", "http://edge/up")
+    end
+
+    def ops_http_public_interval_seconds
+      ENV.fetch("OPS_HTTP_PUBLIC_INTERVAL_SECS", "900").to_i
+    end
+
+    def ops_up_latency_warn_ms
+      ENV.fetch("OPS_UP_LATENCY_WARN_MS", "2000").to_i
+    end
+
+    def ops_up_latency_crit_ms
+      ENV.fetch("OPS_UP_LATENCY_CRIT_MS", "5000").to_i
+    end
+
     def google_analytics_measurement_id
       ENV["GOOGLE_ANALYTICS_MEASUREMENT_ID"].presence
     end

backend/spec/services/ops/health_checks_spec.rb

@@ -24,7 +24,7 @@ RSpec.describe Ops::HealthChecks do
       )
       %i[
         check_recordings_size check_postgres check_redis check_mediamtx check_garage
-        check_sidekiq_heartbeat check_sidekiq_dead check_http_public
+        check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
       ].each do |method|
         allow_any_instance_of(described_class).to receive(method).and_return(
           described_class::Finding.new(
@@ -33,6 +33,7 @@ RSpec.describe Ops::HealthChecks do
           )
         )
       end
+      allow_any_instance_of(described_class).to receive(:public_check_due?).and_return(false)
 
       described_class.new.call
 
@@ -50,7 +51,7 @@ RSpec.describe Ops::HealthChecks do
       )
       %i[
         check_recordings_size check_postgres check_redis check_mediamtx check_garage
-        check_sidekiq_heartbeat check_sidekiq_dead check_http_public
+        check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
       ].each do |method|
         allow_any_instance_of(described_class).to receive(method).and_return(
           described_class::Finding.new(
@@ -59,11 +60,40 @@ RSpec.describe Ops::HealthChecks do
           )
         )
       end
+      allow_any_instance_of(described_class).to receive(:public_check_due?).and_return(false)
 
       summary = described_class.new.summary
 
       expect(summary[:status]).to eq("ok")
-      expect(summary[:checks].size).to eq(9)
+      expect(summary[:checks].size).to eq(10)
+    end
+
+    it "degraded quando la latenza p95 supera la soglia warning" do
+      allow_any_instance_of(described_class).to receive(:build_findings).and_return([
+        described_class::Finding.new(
+          kind: "rails_latency", severity: "warning", healthy: false,
+          title: "Latenza elevata", message: "p95 2500ms", metadata: {}, fingerprint: "rails_latency:p95"
+        )
+      ])
+
+      expect(described_class.new.summary[:status]).to eq("degraded")
+    end
+  end
+
+  describe "#check_http_public" do
+    it "è warning, non critical" do
+      allow(Rails.env).to receive(:test?).and_return(false)
+      allow(MatchLiveTv).to receive(:app_public_url).and_return("https://www.matchlivetv.it")
+
+      checks = described_class.new
+      response = instance_double(Faraday::Response, status: 200)
+      allow(Faraday).to receive(:get).and_return(response)
+
+      finding = checks.send(:check_http_public)
+
+      expect(finding.severity).to eq("warning")
+      expect(finding.kind).to eq("http_public")
+      expect(finding.healthy).to be(true)
     end
   end
 end

backend/spec/services/ops/up_latency_tracker_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Ops::UpLatencyTracker do
+  before do
+    described_class.redis&.del(described_class::REDIS_KEY)
+  end
+
+  it "calcola il p95 sui campioni registrati" do
+    (1..20).each { |ms| described_class.record(ms * 10) }
+
+    expect(described_class.p95).to eq(190)
+  end
+
+  it "restituisce nil senza campioni" do
+    expect(described_class.p95).to be_nil
+  end
+end