eminux/MatchLiveTv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HLS su edge e monitoraggio ops con latenza p95.

Browse Source 
Evita di saturare Puma in diretta, separa i check Rails/pubblico e traccia la latenza /up per alert più affidabili.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Emiliano Frascaro
2026-06-14 21:19:32 +02:00
parent 13390d8a3c
commit a44d9fbadd
11 changed files with 290 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/app/models/ops/incident.rb
View File

@@ -3,8 +3,8 @@ module Ops
self.table_name = "ops_incidents"
KINDS = %w[
disk_space recordings_size service_down http_public sidekiq_stale sidekiq_dead
log_pattern garage_storage
disk_space recordings_size service_down http_public http_rails rails_latency
sidekiq_stale sidekiq_dead log_pattern garage_storage
].freeze
SEVERITIES = %w[critical warning info].freeze
STATUSES = %w[open acknowledged resolved].freeze

166
backend/app/services/ops/health_checks.rb
View File

@@ -9,48 +9,12 @@ module Ops
SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
def call
findings = [
check_disk_root,
check_recordings_size,
check_postgres,
check_redis,
check_mediamtx,
check_garage,
check_sidekiq_heartbeat,
check_sidekiq_dead,
check_http_public
]
findings.each do |finding|
if finding.healthy
Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
else
Ops::IncidentRecorder.record(
kind: finding.kind,
severity: finding.severity,
title: finding.title,
message: finding.message,
metadata: finding.metadata,
fingerprint: finding.fingerprint
)
end
end
findings
build_findings.each { |finding| process_finding(finding) }
build_findings
end
def summary
findings = [
check_disk_root,
check_recordings_size,
check_postgres,
check_redis,
check_mediamtx,
check_garage,
check_sidekiq_heartbeat,
check_sidekiq_dead,
check_http_public
]
findings = build_findings
status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
"down"
@@ -69,6 +33,60 @@ module Ops
private
def build_findings
findings = [
check_disk_root,
check_recordings_size,
check_postgres,
check_redis,
check_mediamtx,
check_garage,
check_sidekiq_heartbeat,
check_sidekiq_dead,
check_http_rails,
check_rails_latency
]
findings << check_http_public if public_check_due?
findings
end
def process_finding(finding)
if finding.healthy
Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
else
Ops::IncidentRecorder.record(
kind: finding.kind,
severity: finding.severity,
title: finding.title,
message: finding.message,
metadata: finding.metadata,
fingerprint: finding.fingerprint
)
end
end
def public_check_due?
return false if Rails.env.test?
redis = health_redis
return true unless redis
last = redis.get(public_check_redis_key).to_i
due = last.zero? || (Time.current.to_i - last) >= MatchLiveTv.ops_http_public_interval_seconds
redis.set(public_check_redis_key, Time.current.to_i) if due
due
end
def public_check_redis_key
"ops:health:last_public_check_at"
end
def health_redis
@health_redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
rescue Redis::CannotConnectError
nil
end
def check_disk_root
disk = Admin::HostMetrics.new.send(:read_disk)[:root]
pct = disk[:used_percent].to_f
@@ -176,24 +194,84 @@ module Ops
)
end
def check_http_rails
return ok_finding("http_rails:skip", "HTTP Rails check disabilitato in test") if Rails.env.test?
url = MatchLiveTv.ops_http_rails_url
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
response = Faraday.get(url) { |req| req.options.timeout = 5 }
elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
Ops::UpLatencyTracker.record(elapsed_ms)
healthy = response.status == 200
Finding.new(
kind: "http_rails",
severity: "critical",
healthy: healthy,
title: healthy ? "Rails raggiungibile via edge" : "Rails non risponde via edge",
message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
fingerprint: "http_rails:up"
)
rescue Faraday::Error => e
fail_finding("http_rails", "critical", "http_rails:up", "Rails non raggiungibile via edge", e.message)
end
def check_rails_latency
return ok_finding("rails_latency:skip", "Latenza Rails non valutata in test") if Rails.env.test?
min_samples = ENV.fetch("OPS_UP_LATENCY_MIN_SAMPLES", "5").to_i
p95 = Ops::UpLatencyTracker.p95
count = Ops::UpLatencyTracker.sample_count
if p95.nil? || count < min_samples
return ok_finding(
"rails_latency:pending",
"Latenza /up: campioni insufficienti (#{count}/#{min_samples})"
)
end
warn_ms = MatchLiveTv.ops_up_latency_warn_ms
crit_ms = MatchLiveTv.ops_up_latency_crit_ms
healthy = p95 < warn_ms
severity = if p95 >= crit_ms
"critical"
elsif p95 >= warn_ms
"warning"
else
"info"
end
Finding.new(
kind: "rails_latency",
severity: severity,
healthy: healthy,
title: healthy ? "Latenza Rails OK" : "Latenza Rails elevata",
message: "p95 GET /up via edge: #{p95}ms (soglia warning #{warn_ms}ms, critical #{crit_ms}ms, n=#{count})",
metadata: { "p95_ms" => p95, "samples" => count, "warn_ms" => warn_ms, "crit_ms" => crit_ms },
fingerprint: "rails_latency:p95"
)
end
def check_http_public
return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
response = Faraday.get(url) { |req| req.options.timeout = 10 }
elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
healthy = response.status == 200
Finding.new(
kind: "http_public",
severity: "critical",
severity: "warning",
healthy: healthy,
title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
metadata: { "url" => url, "status" => response.status },
title: healthy ? "Percorso pubblico raggiungibile" : "Percorso pubblico non risponde",
message: healthy ? "GET #{url} → 200 (#{elapsed_ms}ms)" : "GET #{url} → #{response.status}",
metadata: { "url" => url, "status" => response.status, "latency_ms" => elapsed_ms },
fingerprint: "http_public:up"
)
rescue Faraday::Error => e
fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
fail_finding("http_public", "warning", "http_public:up", "Percorso pubblico non raggiungibile", e.message)
end
def ok_finding(fingerprint, title)

44
backend/app/services/ops/up_latency_tracker.rb Normal file
View File

@@ -0,0 +1,44 @@
# frozen_string_literal: true
module Ops
# Campioni latenza GET /up (edge → rails) per calcolo p95.
class UpLatencyTracker
REDIS_KEY = "ops:up_latency_ms"
MAX_SAMPLES = -> { ENV.fetch("OPS_UP_LATENCY_SAMPLES", "60").to_i }
class << self
def record(latency_ms)
return unless redis
redis.pipelined do |pipe|
pipe.lpush(REDIS_KEY, latency_ms.to_i)
pipe.ltrim(REDIS_KEY, 0, MAX_SAMPLES.call - 1)
end
end
def samples
return [] unless redis
redis.lrange(REDIS_KEY, 0, -1).map(&:to_i)
end
def sample_count
samples.size
end
def p95
sorted = samples.sort
return nil if sorted.empty?
index = [(sorted.size * 0.95).ceil - 1, 0].max
sorted[index]
end
def redis
@redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
rescue Redis::CannotConnectError
nil
end
end
end
end

4
backend/app/services/streams/youtube_relay.rb
View File

@@ -153,8 +153,8 @@ module Streams
end
path = session.mediamtx_path_name
rails = ENV.fetch("RAILS_INTERNAL_URL", "http://rails:3000")
[:hls, "#{rails.chomp('/')}/hls/#{path}/index.m3u8"]
hls = ENV.fetch("MEDIAMTX_HLS_URL", "http://mediamtx:8888").chomp("/")
[:hls, "#{hls}/#{path}/index.m3u8"]
end
def intake_available?(session)