From a44d9fbadd7026936e13a404efbb918639f5f8f9 Mon Sep 17 00:00:00 2001 From: Emiliano Frascaro Date: Sun, 14 Jun 2026 21:19:32 +0200 Subject: [PATCH] HLS su edge e monitoraggio ops con latenza p95. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evita di saturare Puma in diretta, separa i check Rails/pubblico e traccia la latenza /up per alert più affidabili. Co-authored-by: Cursor --- backend/app/models/ops/incident.rb | 4 +- backend/app/services/ops/health_checks.rb | 166 +++++++++++++----- .../app/services/ops/up_latency_tracker.rb | 44 +++++ backend/app/services/streams/youtube_relay.rb | 4 +- backend/config/initializers/match_live_tv.rb | 16 ++ .../spec/services/ops/health_checks_spec.rb | 36 +++- .../services/ops/up_latency_tracker_spec.rb | 19 ++ docs/OPS_MONITORING.md | 8 +- infra/.env.production.example | 8 + infra/docker-compose.prod.yml | 12 ++ infra/nginx-edge/nginx.conf | 25 +++ 11 files changed, 290 insertions(+), 52 deletions(-) create mode 100644 backend/app/services/ops/up_latency_tracker.rb create mode 100644 backend/spec/services/ops/up_latency_tracker_spec.rb diff --git a/backend/app/models/ops/incident.rb b/backend/app/models/ops/incident.rb index edd6e7d..de88ec5 100644 --- a/backend/app/models/ops/incident.rb +++ b/backend/app/models/ops/incident.rb @@ -3,8 +3,8 @@ module Ops self.table_name = "ops_incidents" KINDS = %w[ - disk_space recordings_size service_down http_public sidekiq_stale sidekiq_dead - log_pattern garage_storage + disk_space recordings_size service_down http_public http_rails rails_latency + sidekiq_stale sidekiq_dead log_pattern garage_storage ].freeze SEVERITIES = %w[critical warning info].freeze STATUSES = %w[open acknowledged resolved].freeze diff --git a/backend/app/services/ops/health_checks.rb b/backend/app/services/ops/health_checks.rb index 0043deb..95af0a0 100644 --- a/backend/app/services/ops/health_checks.rb +++ b/backend/app/services/ops/health_checks.rb @@ -9,48 +9,12 @@ module Ops SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i } def call - findings = [ - check_disk_root, - check_recordings_size, - check_postgres, - check_redis, - check_mediamtx, - check_garage, - check_sidekiq_heartbeat, - check_sidekiq_dead, - check_http_public - ] - - findings.each do |finding| - if finding.healthy - Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint) - else - Ops::IncidentRecorder.record( - kind: finding.kind, - severity: finding.severity, - title: finding.title, - message: finding.message, - metadata: finding.metadata, - fingerprint: finding.fingerprint - ) - end - end - - findings + build_findings.each { |finding| process_finding(finding) } + build_findings end def summary - findings = [ - check_disk_root, - check_recordings_size, - check_postgres, - check_redis, - check_mediamtx, - check_garage, - check_sidekiq_heartbeat, - check_sidekiq_dead, - check_http_public - ] + findings = build_findings status = if findings.any? { |f| !f.healthy && f.severity == "critical" } "down" @@ -69,6 +33,60 @@ module Ops private + def build_findings + findings = [ + check_disk_root, + check_recordings_size, + check_postgres, + check_redis, + check_mediamtx, + check_garage, + check_sidekiq_heartbeat, + check_sidekiq_dead, + check_http_rails, + check_rails_latency + ] + findings << check_http_public if public_check_due? + findings + end + + def process_finding(finding) + if finding.healthy + Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint) + else + Ops::IncidentRecorder.record( + kind: finding.kind, + severity: finding.severity, + title: finding.title, + message: finding.message, + metadata: finding.metadata, + fingerprint: finding.fingerprint + ) + end + end + + def public_check_due? + return false if Rails.env.test? + + redis = health_redis + return true unless redis + + last = redis.get(public_check_redis_key).to_i + due = last.zero? || (Time.current.to_i - last) >= MatchLiveTv.ops_http_public_interval_seconds + redis.set(public_check_redis_key, Time.current.to_i) if due + due + end + + def public_check_redis_key + "ops:health:last_public_check_at" + end + + def health_redis + @health_redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) + rescue Redis::CannotConnectError + nil + end + def check_disk_root disk = Admin::HostMetrics.new.send(:read_disk)[:root] pct = disk[:used_percent].to_f @@ -176,24 +194,84 @@ module Ops ) end + def check_http_rails + return ok_finding("http_rails:skip", "HTTP Rails check disabilitato in test") if Rails.env.test? + + url = MatchLiveTv.ops_http_rails_url + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + response = Faraday.get(url) { |req| req.options.timeout = 5 } + elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round + Ops::UpLatencyTracker.record(elapsed_ms) + + healthy = response.status == 200 + Finding.new( + kind: "http_rails", + severity: "critical", + healthy: healthy, + title: healthy ? "Rails raggiungibile via edge" : "Rails non risponde via edge", + message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}", + metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms }, + fingerprint: "http_rails:up" + ) + rescue Faraday::Error => e + fail_finding("http_rails", "critical", "http_rails:up", "Rails non raggiungibile via edge", e.message) + end + + def check_rails_latency + return ok_finding("rails_latency:skip", "Latenza Rails non valutata in test") if Rails.env.test? + + min_samples = ENV.fetch("OPS_UP_LATENCY_MIN_SAMPLES", "5").to_i + p95 = Ops::UpLatencyTracker.p95 + count = Ops::UpLatencyTracker.sample_count + if p95.nil? || count < min_samples + return ok_finding( + "rails_latency:pending", + "Latenza /up: campioni insufficienti (#{count}/#{min_samples})" + ) + end + + warn_ms = MatchLiveTv.ops_up_latency_warn_ms + crit_ms = MatchLiveTv.ops_up_latency_crit_ms + healthy = p95 < warn_ms + severity = if p95 >= crit_ms + "critical" + elsif p95 >= warn_ms + "warning" + else + "info" + end + + Finding.new( + kind: "rails_latency", + severity: severity, + healthy: healthy, + title: healthy ? "Latenza Rails OK" : "Latenza Rails elevata", + message: "p95 GET /up via edge: #{p95}ms (soglia warning #{warn_ms}ms, critical #{crit_ms}ms, n=#{count})", + metadata: { "p95_ms" => p95, "samples" => count, "warn_ms" => warn_ms, "crit_ms" => crit_ms }, + fingerprint: "rails_latency:p95" + ) + end + def check_http_public return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test? url = "#{MatchLiveTv.app_public_url.chomp('/')}/up" + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) response = Faraday.get(url) { |req| req.options.timeout = 10 } + elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round healthy = response.status == 200 Finding.new( kind: "http_public", - severity: "critical", + severity: "warning", healthy: healthy, - title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde", - message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}", - metadata: { "url" => url, "status" => response.status }, + title: healthy ? "Percorso pubblico raggiungibile" : "Percorso pubblico non risponde", + message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}", + metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms }, fingerprint: "http_public:up" ) rescue Faraday::Error => e - fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message) + fail_finding("http_public", "warning", "http_public:up", "Percorso pubblico non raggiungibile", e.message) end def ok_finding(fingerprint, title) diff --git a/backend/app/services/ops/up_latency_tracker.rb b/backend/app/services/ops/up_latency_tracker.rb new file mode 100644 index 0000000..c213a4c --- /dev/null +++ b/backend/app/services/ops/up_latency_tracker.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Ops + # Campioni latenza GET /up (edge → rails) per calcolo p95. + class UpLatencyTracker + REDIS_KEY = "ops:up_latency_ms" + MAX_SAMPLES = -> { ENV.fetch("OPS_UP_LATENCY_SAMPLES", "60").to_i } + + class << self + def record(latency_ms) + return unless redis + + redis.pipelined do |pipe| + pipe.lpush(REDIS_KEY, latency_ms.to_i) + pipe.ltrim(REDIS_KEY, 0, MAX_SAMPLES.call - 1) + end + end + + def samples + return [] unless redis + + redis.lrange(REDIS_KEY, 0, -1).map(&:to_i) + end + + def sample_count + samples.size + end + + def p95 + sorted = samples.sort + return nil if sorted.empty? + + index = [(sorted.size * 0.95).ceil - 1, 0].max + sorted[index] + end + + def redis + @redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) + rescue Redis::CannotConnectError + nil + end + end + end +end diff --git a/backend/app/services/streams/youtube_relay.rb b/backend/app/services/streams/youtube_relay.rb index 71d8871..24f80c2 100644 --- a/backend/app/services/streams/youtube_relay.rb +++ b/backend/app/services/streams/youtube_relay.rb @@ -153,8 +153,8 @@ module Streams end path = session.mediamtx_path_name - rails = ENV.fetch("RAILS_INTERNAL_URL", "http://rails:3000") - [:hls, "#{rails.chomp('/')}/hls/#{path}/index.m3u8"] + hls = ENV.fetch("MEDIAMTX_HLS_URL", "http://mediamtx:8888").chomp("/") + [:hls, "#{hls}/#{path}/index.m3u8"] end def intake_available?(session) diff --git a/backend/config/initializers/match_live_tv.rb b/backend/config/initializers/match_live_tv.rb index 1431762..2daf48d 100644 --- a/backend/config/initializers/match_live_tv.rb +++ b/backend/config/initializers/match_live_tv.rb @@ -151,6 +151,22 @@ module MatchLiveTv ENV.fetch("REPLAY_MEDIA_REDIRECT", "true") == "true" end + def ops_http_rails_url + ENV.fetch("OPS_HTTP_RAILS_URL", "http://edge/up") + end + + def ops_http_public_interval_seconds + ENV.fetch("OPS_HTTP_PUBLIC_INTERVAL_SECS", "900").to_i + end + + def ops_up_latency_warn_ms + ENV.fetch("OPS_UP_LATENCY_WARN_MS", "2000").to_i + end + + def ops_up_latency_crit_ms + ENV.fetch("OPS_UP_LATENCY_CRIT_MS", "5000").to_i + end + def google_analytics_measurement_id ENV["GOOGLE_ANALYTICS_MEASUREMENT_ID"].presence end diff --git a/backend/spec/services/ops/health_checks_spec.rb b/backend/spec/services/ops/health_checks_spec.rb index 9032884..e4f7f62 100644 --- a/backend/spec/services/ops/health_checks_spec.rb +++ b/backend/spec/services/ops/health_checks_spec.rb @@ -24,7 +24,7 @@ RSpec.describe Ops::HealthChecks do ) %i[ check_recordings_size check_postgres check_redis check_mediamtx check_garage - check_sidekiq_heartbeat check_sidekiq_dead check_http_public + check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency ].each do |method| allow_any_instance_of(described_class).to receive(method).and_return( described_class::Finding.new( @@ -33,6 +33,7 @@ RSpec.describe Ops::HealthChecks do ) ) end + allow_any_instance_of(described_class).to receive(:public_check_due?).and_return(false) described_class.new.call @@ -50,7 +51,7 @@ RSpec.describe Ops::HealthChecks do ) %i[ check_recordings_size check_postgres check_redis check_mediamtx check_garage - check_sidekiq_heartbeat check_sidekiq_dead check_http_public + check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency ].each do |method| allow_any_instance_of(described_class).to receive(method).and_return( described_class::Finding.new( @@ -59,11 +60,40 @@ RSpec.describe Ops::HealthChecks do ) ) end + allow_any_instance_of(described_class).to receive(:public_check_due?).and_return(false) summary = described_class.new.summary expect(summary[:status]).to eq("ok") - expect(summary[:checks].size).to eq(9) + expect(summary[:checks].size).to eq(10) + end + + it "degraded quando la latenza p95 supera la soglia warning" do + allow_any_instance_of(described_class).to receive(:build_findings).and_return([ + described_class::Finding.new( + kind: "rails_latency", severity: "warning", healthy: false, + title: "Latenza elevata", message: "p95 2500ms", metadata: {}, fingerprint: "rails_latency:p95" + ) + ]) + + expect(described_class.new.summary[:status]).to eq("degraded") + end + end + + describe "#check_http_public" do + it "è warning, non critical" do + allow(Rails.env).to receive(:test?).and_return(false) + allow(MatchLiveTv).to receive(:app_public_url).and_return("https://www.matchlivetv.it") + + checks = described_class.new + response = instance_double(Faraday::Response, status: 200) + allow(Faraday).to receive(:get).and_return(response) + + finding = checks.send(:check_http_public) + + expect(finding.severity).to eq("warning") + expect(finding.kind).to eq("http_public") + expect(finding.healthy).to be(true) end end end diff --git a/backend/spec/services/ops/up_latency_tracker_spec.rb b/backend/spec/services/ops/up_latency_tracker_spec.rb new file mode 100644 index 0000000..9a38a1e --- /dev/null +++ b/backend/spec/services/ops/up_latency_tracker_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Ops::UpLatencyTracker do + before do + described_class.redis&.del(described_class::REDIS_KEY) + end + + it "calcola il p95 sui campioni registrati" do + (1..20).each { |ms| described_class.record(ms * 10) } + + expect(described_class.p95).to eq(190) + end + + it "restituisce nil senza campioni" do + expect(described_class.p95).to be_nil + end +end diff --git a/docs/OPS_MONITORING.md b/docs/OPS_MONITORING.md index e4a0a68..b0a60fa 100644 --- a/docs/OPS_MONITORING.md +++ b/docs/OPS_MONITORING.md @@ -7,7 +7,7 @@ Sistema integrato per health check continui, analisi log, dashboard incidenti e | Componente | Ruolo | |------------|-------| | `Ops::HealthMonitorJob` | Sidekiq — health check ogni ~3 min | -| `Ops::HealthChecks` | Disco, DB, Redis, MediaMTX, Garage, Sidekiq, HTTP pubblico | +| `Ops::HealthChecks` | Disco, DB, Redis, MediaMTX, Garage, Sidekiq, HTTP Rails (critical), latenza p95 `/up`, HTTP pubblico (warning) | | `Ops::LogScanner` | Pattern regex su log Docker (cron ogni 10 min) | | `Ops::Incident` | DB — incidenti con deduplicazione | | `Ops::Notifier` | Push ntfy + email opzionale | @@ -81,6 +81,12 @@ Vedi [`infra/.env.production.example`](../infra/.env.production.example). | `OPS_NTFY_TOKEN` | — | Bearer token (topic protetto) | | `OPS_ALERT_EMAIL` | — | Fallback email | | `OPS_HEALTH_INTERVAL_SECS` | 180 | Intervallo health job | +| `OPS_HTTP_RAILS_URL` | `http://edge/up` | Check Rails via edge (no hairpin NPM) | +| `OPS_HTTP_PUBLIC_INTERVAL_SECS` | 900 | Intervallo check URL pubblico (warning) | +| `OPS_UP_LATENCY_WARN_MS` | 2000 | Soglia warning latenza p95 `/up` | +| `OPS_UP_LATENCY_CRIT_MS` | 5000 | Soglia critical latenza p95 `/up` | +| `OPS_UP_LATENCY_SAMPLES` | 60 | Campioni Redis per p95 | +| `OPS_UP_LATENCY_MIN_SAMPLES` | 5 | Campioni minimi prima di alert latenza | | `OPS_HEALTH_TOKEN` | — | Token per `/up/deep` | | `OPS_DISK_CRIT_PERCENT` | 90 | Soglia critical disco | | `OPS_LOG_SUBSCRIBER` | false | Cattura 500 in-process | diff --git a/infra/.env.production.example b/infra/.env.production.example index 5755ba4..fd82cae 100644 --- a/infra/.env.production.example +++ b/infra/.env.production.example @@ -73,6 +73,14 @@ REPLAY_MEDIA_REDIRECT=true # Puma: 5 thread (no secondo worker senza più RAM) RAILS_MAX_THREADS=5 +# Ops — check Rails via edge (critical) + percorso pubblico (warning, ogni 15 min) +OPS_HTTP_RAILS_URL=http://edge/up +OPS_HTTP_PUBLIC_INTERVAL_SECS=900 +OPS_UP_LATENCY_WARN_MS=2000 +OPS_UP_LATENCY_CRIT_MS=5000 +OPS_UP_LATENCY_SAMPLES=60 +OPS_UP_LATENCY_MIN_SAMPLES=5 + # Ops monitoring — notifiche push (ntfy self-hosted) e health check # 1) Avvia servizio: docker compose up -d ntfy # 2) NPM: proxy HTTPS → http://192.168.1.146:8090 (es. ntfy.matchlivetv.it, WebSockets ON) diff --git a/infra/docker-compose.prod.yml b/infra/docker-compose.prod.yml index f079a57..c54a336 100644 --- a/infra/docker-compose.prod.yml +++ b/infra/docker-compose.prod.yml @@ -108,6 +108,12 @@ services: OPS_NOTIFY_SEVERITIES: ${OPS_NOTIFY_SEVERITIES:-critical,warning} OPS_NOTIFY_COOLDOWN_MINUTES: ${OPS_NOTIFY_COOLDOWN_MINUTES:-30} OPS_HEALTH_INTERVAL_SECS: ${OPS_HEALTH_INTERVAL_SECS:-180} + OPS_HTTP_RAILS_URL: ${OPS_HTTP_RAILS_URL:-http://edge/up} + OPS_HTTP_PUBLIC_INTERVAL_SECS: ${OPS_HTTP_PUBLIC_INTERVAL_SECS:-900} + OPS_UP_LATENCY_WARN_MS: ${OPS_UP_LATENCY_WARN_MS:-2000} + OPS_UP_LATENCY_CRIT_MS: ${OPS_UP_LATENCY_CRIT_MS:-5000} + OPS_UP_LATENCY_SAMPLES: ${OPS_UP_LATENCY_SAMPLES:-60} + OPS_UP_LATENCY_MIN_SAMPLES: ${OPS_UP_LATENCY_MIN_SAMPLES:-5} OPS_HEALTH_TOKEN: ${OPS_HEALTH_TOKEN:-} OPS_DISK_WARN_PERCENT: ${OPS_DISK_WARN_PERCENT:-80} OPS_DISK_CRIT_PERCENT: ${OPS_DISK_CRIT_PERCENT:-90} @@ -198,6 +204,12 @@ services: OPS_NOTIFY_SEVERITIES: ${OPS_NOTIFY_SEVERITIES:-critical,warning} OPS_NOTIFY_COOLDOWN_MINUTES: ${OPS_NOTIFY_COOLDOWN_MINUTES:-30} OPS_HEALTH_INTERVAL_SECS: ${OPS_HEALTH_INTERVAL_SECS:-180} + OPS_HTTP_RAILS_URL: ${OPS_HTTP_RAILS_URL:-http://edge/up} + OPS_HTTP_PUBLIC_INTERVAL_SECS: ${OPS_HTTP_PUBLIC_INTERVAL_SECS:-900} + OPS_UP_LATENCY_WARN_MS: ${OPS_UP_LATENCY_WARN_MS:-2000} + OPS_UP_LATENCY_CRIT_MS: ${OPS_UP_LATENCY_CRIT_MS:-5000} + OPS_UP_LATENCY_SAMPLES: ${OPS_UP_LATENCY_SAMPLES:-60} + OPS_UP_LATENCY_MIN_SAMPLES: ${OPS_UP_LATENCY_MIN_SAMPLES:-5} OPS_HEALTH_TOKEN: ${OPS_HEALTH_TOKEN:-} OPS_DISK_WARN_PERCENT: ${OPS_DISK_WARN_PERCENT:-80} OPS_DISK_CRIT_PERCENT: ${OPS_DISK_CRIT_PERCENT:-90} diff --git a/infra/nginx-edge/nginx.conf b/infra/nginx-edge/nginx.conf index 17722b8..6d0230b 100644 --- a/infra/nginx-edge/nginx.conf +++ b/infra/nginx-edge/nginx.conf @@ -14,6 +14,11 @@ http { '' close; } + map $http_cookie $hls_cookie { + default $http_cookie; + "" "cookieCheck=1"; + } + server { listen 80; server_name _; @@ -44,6 +49,26 @@ http { proxy_set_header If-Range $http_if_range; } + # HLS live/regia: proxy diretto a MediaMTX (non passa da Puma). + location /hls/ { + limit_except GET HEAD { + deny all; + } + + rewrite ^/hls/(.*)$ /$1 break; + proxy_pass http://mediamtx:8888; + proxy_http_version 1.1; + proxy_buffering off; + proxy_request_buffering off; + proxy_connect_timeout 5s; + proxy_read_timeout 3600s; + proxy_set_header Host $host; + proxy_set_header Cookie $hls_cookie; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + location = /edge-health { access_log off; add_header Content-Type text/plain;