HLS su edge e monitoraggio ops con latenza p95.

Evita di saturare Puma in diretta, separa i check Rails/pubblico e traccia la latenza /up per alert più affidabili.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/models/ops/incident.rb

@@ -3,8 +3,8 @@ module Ops
     self.table_name = "ops_incidents"
 
     KINDS = %w[
-      disk_space recordings_size service_down http_public sidekiq_stale sidekiq_dead
+      disk_space recordings_size service_down http_public http_rails rails_latency
-      log_pattern garage_storage
+      sidekiq_stale sidekiq_dead log_pattern garage_storage
     ].freeze
     SEVERITIES = %w[critical warning info].freeze
     STATUSES = %w[open acknowledged resolved].freeze

backend/app/services/ops/health_checks.rb

@@ -9,48 +9,12 @@ module Ops
     SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
 
     def call
-      findings = [
+      build_findings.each { |finding| process_finding(finding) }
-        check_disk_root,
+      build_findings
-        check_recordings_size,
-        check_postgres,
-        check_redis,
-        check_mediamtx,
-        check_garage,
-        check_sidekiq_heartbeat,
-        check_sidekiq_dead,
-        check_http_public
-      ]
-
-      findings.each do |finding|
-        if finding.healthy
-          Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
-        else
-          Ops::IncidentRecorder.record(
-            kind: finding.kind,
-            severity: finding.severity,
-            title: finding.title,
-            message: finding.message,
-            metadata: finding.metadata,
-            fingerprint: finding.fingerprint
-          )
-        end
-      end
-
-      findings
     end
 
     def summary
-      findings = [
+      findings = build_findings
-        check_disk_root,
-        check_recordings_size,
-        check_postgres,
-        check_redis,
-        check_mediamtx,
-        check_garage,
-        check_sidekiq_heartbeat,
-        check_sidekiq_dead,
-        check_http_public
-      ]
 
       status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
                  "down"
@@ -69,6 +33,60 @@ module Ops
 
     private
 
+    def build_findings
+      findings = [
+        check_disk_root,
+        check_recordings_size,
+        check_postgres,
+        check_redis,
+        check_mediamtx,
+        check_garage,
+        check_sidekiq_heartbeat,
+        check_sidekiq_dead,
+        check_http_rails,
+        check_rails_latency
+      ]
+      findings << check_http_public if public_check_due?
+      findings
+    end
+
+    def process_finding(finding)
+      if finding.healthy
+        Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
+      else
+        Ops::IncidentRecorder.record(
+          kind: finding.kind,
+          severity: finding.severity,
+          title: finding.title,
+          message: finding.message,
+          metadata: finding.metadata,
+          fingerprint: finding.fingerprint
+        )
+      end
+    end
+
+    def public_check_due?
+      return false if Rails.env.test?
+
+      redis = health_redis
+      return true unless redis
+
+      last = redis.get(public_check_redis_key).to_i
+      due = last.zero? || (Time.current.to_i - last) >= MatchLiveTv.ops_http_public_interval_seconds
+      redis.set(public_check_redis_key, Time.current.to_i) if due
+      due
+    end
+
+    def public_check_redis_key
+      "ops:health:last_public_check_at"
+    end
+
+    def health_redis
+      @health_redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+    rescue Redis::CannotConnectError
+      nil
+    end
+
     def check_disk_root
       disk = Admin::HostMetrics.new.send(:read_disk)[:root]
       pct = disk[:used_percent].to_f
@@ -176,24 +194,84 @@ module Ops
       )
     end
 
+    def check_http_rails
+      return ok_finding("http_rails:skip", "HTTP Rails check disabilitato in test") if Rails.env.test?
+
+      url = MatchLiveTv.ops_http_rails_url
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      response = Faraday.get(url) { |req| req.options.timeout = 5 }
+      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
+      Ops::UpLatencyTracker.record(elapsed_ms)
+
+      healthy = response.status == 200
+      Finding.new(
+        kind: "http_rails",
+        severity: "critical",
+        healthy: healthy,
+        title: healthy ? "Rails raggiungibile via edge" : "Rails non risponde via edge",
+        message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
+        metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
+        fingerprint: "http_rails:up"
+      )
+    rescue Faraday::Error => e
+      fail_finding("http_rails", "critical", "http_rails:up", "Rails non raggiungibile via edge", e.message)
+    end
+
+    def check_rails_latency
+      return ok_finding("rails_latency:skip", "Latenza Rails non valutata in test") if Rails.env.test?
+
+      min_samples = ENV.fetch("OPS_UP_LATENCY_MIN_SAMPLES", "5").to_i
+      p95 = Ops::UpLatencyTracker.p95
+      count = Ops::UpLatencyTracker.sample_count
+      if p95.nil? || count < min_samples
+        return ok_finding(
+          "rails_latency:pending",
+          "Latenza /up: campioni insufficienti (#{count}/#{min_samples})"
+        )
+      end
+
+      warn_ms = MatchLiveTv.ops_up_latency_warn_ms
+      crit_ms = MatchLiveTv.ops_up_latency_crit_ms
+      healthy = p95 < warn_ms
+      severity = if p95 >= crit_ms
+                   "critical"
+                 elsif p95 >= warn_ms
+                   "warning"
+                 else
+                   "info"
+                 end
+
+      Finding.new(
+        kind: "rails_latency",
+        severity: severity,
+        healthy: healthy,
+        title: healthy ? "Latenza Rails OK" : "Latenza Rails elevata",
+        message: "p95 GET /up via edge: #{p95}ms (soglia warning #{warn_ms}ms, critical #{crit_ms}ms, n=#{count})",
+        metadata: { "p95_ms" => p95, "samples" => count, "warn_ms" => warn_ms, "crit_ms" => crit_ms },
+        fingerprint: "rails_latency:p95"
+      )
+    end
+
     def check_http_public
       return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
 
       url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       response = Faraday.get(url) { |req| req.options.timeout = 10 }
+      elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
       healthy = response.status == 200
 
       Finding.new(
         kind: "http_public",
-        severity: "critical",
+        severity: "warning",
         healthy: healthy,
-        title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
+        title: healthy ? "Percorso pubblico raggiungibile" : "Percorso pubblico non risponde",
-        message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
+        message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
-        metadata: { "url" => url, "status" => response.status },
+        metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
         fingerprint: "http_public:up"
       )
     rescue Faraday::Error => e
-      fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
+      fail_finding("http_public", "warning", "http_public:up", "Percorso pubblico non raggiungibile", e.message)
     end
 
     def ok_finding(fingerprint, title)

backend/app/services/ops/up_latency_tracker.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Ops
+  # Campioni latenza GET /up (edge → rails) per calcolo p95.
+  class UpLatencyTracker
+    REDIS_KEY = "ops:up_latency_ms"
+    MAX_SAMPLES = -> { ENV.fetch("OPS_UP_LATENCY_SAMPLES", "60").to_i }
+
+    class << self
+      def record(latency_ms)
+        return unless redis
+
+        redis.pipelined do |pipe|
+          pipe.lpush(REDIS_KEY, latency_ms.to_i)
+          pipe.ltrim(REDIS_KEY, 0, MAX_SAMPLES.call - 1)
+        end
+      end
+
+      def samples
+        return [] unless redis
+
+        redis.lrange(REDIS_KEY, 0, -1).map(&:to_i)
+      end
+
+      def sample_count
+        samples.size
+      end
+
+      def p95
+        sorted = samples.sort
+        return nil if sorted.empty?
+
+        index = [(sorted.size * 0.95).ceil - 1, 0].max
+        sorted[index]
+      end
+
+      def redis
+        @redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+      rescue Redis::CannotConnectError
+        nil
+      end
+    end
+  end
+end

backend/app/services/streams/youtube_relay.rb

@@ -153,8 +153,8 @@ module Streams
         end
 
         path = session.mediamtx_path_name
-        rails = ENV.fetch("RAILS_INTERNAL_URL", "http://rails:3000")
+        hls = ENV.fetch("MEDIAMTX_HLS_URL", "http://mediamtx:8888").chomp("/")
-        [:hls, "#{rails.chomp('/')}/hls/#{path}/index.m3u8"]
+        [:hls, "#{hls}/#{path}/index.m3u8"]
       end
 
       def intake_available?(session)

backend/config/initializers/match_live_tv.rb

@@ -151,6 +151,22 @@ module MatchLiveTv
       ENV.fetch("REPLAY_MEDIA_REDIRECT", "true") == "true"
     end
 
+    def ops_http_rails_url
+      ENV.fetch("OPS_HTTP_RAILS_URL", "http://edge/up")
+    end
+
+    def ops_http_public_interval_seconds
+      ENV.fetch("OPS_HTTP_PUBLIC_INTERVAL_SECS", "900").to_i
+    end
+
+    def ops_up_latency_warn_ms
+      ENV.fetch("OPS_UP_LATENCY_WARN_MS", "2000").to_i
+    end
+
+    def ops_up_latency_crit_ms
+      ENV.fetch("OPS_UP_LATENCY_CRIT_MS", "5000").to_i
+    end
+
     def google_analytics_measurement_id
       ENV["GOOGLE_ANALYTICS_MEASUREMENT_ID"].presence
     end

backend/spec/services/ops/health_checks_spec.rb

@@ -24,7 +24,7 @@ RSpec.describe Ops::HealthChecks do
       )
       %i[
         check_recordings_size check_postgres check_redis check_mediamtx check_garage
-        check_sidekiq_heartbeat check_sidekiq_dead check_http_public
+        check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
       ].each do |method|
         allow_any_instance_of(described_class).to receive(method).and_return(
           described_class::Finding.new(
@@ -33,6 +33,7 @@ RSpec.describe Ops::HealthChecks do
           )
         )
       end
+      allow_any_instance_of(described_class).to receive(:public_check_due?).and_return(false)
 
       described_class.new.call
 
@@ -50,7 +51,7 @@ RSpec.describe Ops::HealthChecks do
       )
       %i[
         check_recordings_size check_postgres check_redis check_mediamtx check_garage
-        check_sidekiq_heartbeat check_sidekiq_dead check_http_public
+        check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
       ].each do |method|
         allow_any_instance_of(described_class).to receive(method).and_return(
           described_class::Finding.new(
@@ -59,11 +60,40 @@ RSpec.describe Ops::HealthChecks do
           )
         )
       end
+      allow_any_instance_of(described_class).to receive(:public_check_due?).and_return(false)
 
       summary = described_class.new.summary
 
       expect(summary[:status]).to eq("ok")
-      expect(summary[:checks].size).to eq(9)
+      expect(summary[:checks].size).to eq(10)
+    end
+
+    it "degraded quando la latenza p95 supera la soglia warning" do
+      allow_any_instance_of(described_class).to receive(:build_findings).and_return([
+        described_class::Finding.new(
+          kind: "rails_latency", severity: "warning", healthy: false,
+          title: "Latenza elevata", message: "p95 2500ms", metadata: {}, fingerprint: "rails_latency:p95"
+        )
+      ])
+
+      expect(described_class.new.summary[:status]).to eq("degraded")
+    end
+  end
+
+  describe "#check_http_public" do
+    it "è warning, non critical" do
+      allow(Rails.env).to receive(:test?).and_return(false)
+      allow(MatchLiveTv).to receive(:app_public_url).and_return("https://www.matchlivetv.it")
+
+      checks = described_class.new
+      response = instance_double(Faraday::Response, status: 200)
+      allow(Faraday).to receive(:get).and_return(response)
+
+      finding = checks.send(:check_http_public)
+
+      expect(finding.severity).to eq("warning")
+      expect(finding.kind).to eq("http_public")
+      expect(finding.healthy).to be(true)
     end
   end
 end

backend/spec/services/ops/up_latency_tracker_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Ops::UpLatencyTracker do
+  before do
+    described_class.redis&.del(described_class::REDIS_KEY)
+  end
+
+  it "calcola il p95 sui campioni registrati" do
+    (1..20).each { |ms| described_class.record(ms * 10) }
+
+    expect(described_class.p95).to eq(190)
+  end
+
+  it "restituisce nil senza campioni" do
+    expect(described_class.p95).to be_nil
+  end
+end

docs/OPS_MONITORING.md

@@ -7,7 +7,7 @@ Sistema integrato per health check continui, analisi log, dashboard incidenti e
 | Componente | Ruolo |
 |------------|-------|
 | `Ops::HealthMonitorJob` | Sidekiq — health check ogni ~3 min |
-| `Ops::HealthChecks` | Disco, DB, Redis, MediaMTX, Garage, Sidekiq, HTTP pubblico |
+| `Ops::HealthChecks` | Disco, DB, Redis, MediaMTX, Garage, Sidekiq, HTTP Rails (critical), latenza p95 `/up`, HTTP pubblico (warning) |
 | `Ops::LogScanner` | Pattern regex su log Docker (cron ogni 10 min) |
 | `Ops::Incident` | DB — incidenti con deduplicazione |
 | `Ops::Notifier` | Push ntfy + email opzionale |
@@ -81,6 +81,12 @@ Vedi [`infra/.env.production.example`](../infra/.env.production.example).
 | `OPS_NTFY_TOKEN` | — | Bearer token (topic protetto) |
 | `OPS_ALERT_EMAIL` | — | Fallback email |
 | `OPS_HEALTH_INTERVAL_SECS` | 180 | Intervallo health job |
+| `OPS_HTTP_RAILS_URL` | `http://edge/up` | Check Rails via edge (no hairpin NPM) |
+| `OPS_HTTP_PUBLIC_INTERVAL_SECS` | 900 | Intervallo check URL pubblico (warning) |
+| `OPS_UP_LATENCY_WARN_MS` | 2000 | Soglia warning latenza p95 `/up` |
+| `OPS_UP_LATENCY_CRIT_MS` | 5000 | Soglia critical latenza p95 `/up` |
+| `OPS_UP_LATENCY_SAMPLES` | 60 | Campioni Redis per p95 |
+| `OPS_UP_LATENCY_MIN_SAMPLES` | 5 | Campioni minimi prima di alert latenza |
 | `OPS_HEALTH_TOKEN` | — | Token per `/up/deep` |
 | `OPS_DISK_CRIT_PERCENT` | 90 | Soglia critical disco |
 | `OPS_LOG_SUBSCRIBER` | false | Cattura 500 in-process |

infra/.env.production.example

@@ -73,6 +73,14 @@ REPLAY_MEDIA_REDIRECT=true
 # Puma: 5 thread (no secondo worker senza più RAM)
 RAILS_MAX_THREADS=5
 
+# Ops — check Rails via edge (critical) + percorso pubblico (warning, ogni 15 min)
+OPS_HTTP_RAILS_URL=http://edge/up
+OPS_HTTP_PUBLIC_INTERVAL_SECS=900
+OPS_UP_LATENCY_WARN_MS=2000
+OPS_UP_LATENCY_CRIT_MS=5000
+OPS_UP_LATENCY_SAMPLES=60
+OPS_UP_LATENCY_MIN_SAMPLES=5
+
 # Ops monitoring — notifiche push (ntfy self-hosted) e health check
 # 1) Avvia servizio: docker compose up -d ntfy
 # 2) NPM: proxy HTTPS → http://192.168.1.146:8090 (es. ntfy.matchlivetv.it, WebSockets ON)

infra/docker-compose.prod.yml

@@ -108,6 +108,12 @@ services:
       OPS_NOTIFY_SEVERITIES: ${OPS_NOTIFY_SEVERITIES:-critical,warning}
       OPS_NOTIFY_COOLDOWN_MINUTES: ${OPS_NOTIFY_COOLDOWN_MINUTES:-30}
       OPS_HEALTH_INTERVAL_SECS: ${OPS_HEALTH_INTERVAL_SECS:-180}
+      OPS_HTTP_RAILS_URL: ${OPS_HTTP_RAILS_URL:-http://edge/up}
+      OPS_HTTP_PUBLIC_INTERVAL_SECS: ${OPS_HTTP_PUBLIC_INTERVAL_SECS:-900}
+      OPS_UP_LATENCY_WARN_MS: ${OPS_UP_LATENCY_WARN_MS:-2000}
+      OPS_UP_LATENCY_CRIT_MS: ${OPS_UP_LATENCY_CRIT_MS:-5000}
+      OPS_UP_LATENCY_SAMPLES: ${OPS_UP_LATENCY_SAMPLES:-60}
+      OPS_UP_LATENCY_MIN_SAMPLES: ${OPS_UP_LATENCY_MIN_SAMPLES:-5}
       OPS_HEALTH_TOKEN: ${OPS_HEALTH_TOKEN:-}
       OPS_DISK_WARN_PERCENT: ${OPS_DISK_WARN_PERCENT:-80}
       OPS_DISK_CRIT_PERCENT: ${OPS_DISK_CRIT_PERCENT:-90}
@@ -198,6 +204,12 @@ services:
       OPS_NOTIFY_SEVERITIES: ${OPS_NOTIFY_SEVERITIES:-critical,warning}
       OPS_NOTIFY_COOLDOWN_MINUTES: ${OPS_NOTIFY_COOLDOWN_MINUTES:-30}
       OPS_HEALTH_INTERVAL_SECS: ${OPS_HEALTH_INTERVAL_SECS:-180}
+      OPS_HTTP_RAILS_URL: ${OPS_HTTP_RAILS_URL:-http://edge/up}
+      OPS_HTTP_PUBLIC_INTERVAL_SECS: ${OPS_HTTP_PUBLIC_INTERVAL_SECS:-900}
+      OPS_UP_LATENCY_WARN_MS: ${OPS_UP_LATENCY_WARN_MS:-2000}
+      OPS_UP_LATENCY_CRIT_MS: ${OPS_UP_LATENCY_CRIT_MS:-5000}
+      OPS_UP_LATENCY_SAMPLES: ${OPS_UP_LATENCY_SAMPLES:-60}
+      OPS_UP_LATENCY_MIN_SAMPLES: ${OPS_UP_LATENCY_MIN_SAMPLES:-5}
       OPS_HEALTH_TOKEN: ${OPS_HEALTH_TOKEN:-}
       OPS_DISK_WARN_PERCENT: ${OPS_DISK_WARN_PERCENT:-80}
       OPS_DISK_CRIT_PERCENT: ${OPS_DISK_CRIT_PERCENT:-90}

infra/nginx-edge/nginx.conf

@@ -14,6 +14,11 @@ http {
         ''      close;
     }
 
+    map $http_cookie $hls_cookie {
+        default $http_cookie;
+        ""      "cookieCheck=1";
+    }
+
     server {
         listen 80;
         server_name _;
@@ -44,6 +49,26 @@ http {
             proxy_set_header If-Range $http_if_range;
         }
 
+        # HLS live/regia: proxy diretto a MediaMTX (non passa da Puma).
+        location /hls/ {
+            limit_except GET HEAD {
+                deny all;
+            }
+
+            rewrite ^/hls/(.*)$ /$1 break;
+            proxy_pass http://mediamtx:8888;
+            proxy_http_version 1.1;
+            proxy_buffering off;
+            proxy_request_buffering off;
+            proxy_connect_timeout 5s;
+            proxy_read_timeout 3600s;
+            proxy_set_header Host $host;
+            proxy_set_header Cookie $hls_cookie;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
         location = /edge-health {
             access_log off;
             add_header Content-Type text/plain;