Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -19,6 +19,8 @@ gem "faraday"
|
||||
gem "stripe"
|
||||
gem "aws-sdk-s3", "~> 1.0", require: false
|
||||
gem "kamal", require: false
|
||||
gem "sentry-ruby"
|
||||
gem "sentry-rails"
|
||||
|
||||
group :development, :test do
|
||||
gem "debug", platforms: %i[ mri windows ], require: "debug/prelude"
|
||||
|
||||
@@ -381,6 +381,13 @@ GEM
|
||||
rubocop-rails (>= 2.30)
|
||||
ruby-progressbar (1.13.0)
|
||||
securerandom (0.4.1)
|
||||
sentry-rails (6.6.2)
|
||||
railties (>= 5.2.0)
|
||||
sentry-ruby (~> 6.6.2)
|
||||
sentry-ruby (6.6.2)
|
||||
bigdecimal
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
logger
|
||||
sidekiq (8.1.5)
|
||||
connection_pool (>= 3.0.0)
|
||||
json (>= 2.16.0)
|
||||
@@ -456,6 +463,8 @@ DEPENDENCIES
|
||||
redis (>= 4.0.1)
|
||||
rspec-rails
|
||||
rubocop-rails-omakase
|
||||
sentry-rails
|
||||
sentry-ruby
|
||||
sidekiq
|
||||
stripe
|
||||
tzinfo-data
|
||||
|
||||
@@ -13,7 +13,22 @@ module Admin
|
||||
def metrics
|
||||
host = HostMetrics.new.sample!
|
||||
stats = DashboardStats.new.call
|
||||
render json: { host: host, stats: stats, at: Time.current.iso8601 }
|
||||
render json: {
|
||||
host: host,
|
||||
stats: stats,
|
||||
ops_summary: ops_summary,
|
||||
at: Time.current.iso8601
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ops_summary
|
||||
{
|
||||
open_critical: Ops::Incident.critical_open.count,
|
||||
open_warning: Ops::Incident.open.where(severity: "warning").count,
|
||||
open_total: Ops::Incident.open.count
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
37
backend/app/controllers/admin/ops_controller.rb
Normal file
37
backend/app/controllers/admin/ops_controller.rb
Normal file
@@ -0,0 +1,37 @@
|
||||
module Admin
|
||||
class OpsController < Admin::BaseController
|
||||
def index
|
||||
@open_incidents = Ops::Incident.open.recent.limit(50)
|
||||
@resolved_incidents = Ops::Incident.where(status: "resolved").since(30.days.ago).recent.limit(30)
|
||||
@summary = ops_summary
|
||||
end
|
||||
|
||||
def acknowledge
|
||||
incident = Ops::Incident.find(params[:id])
|
||||
incident.acknowledge!
|
||||
redirect_to admin_ops_path, notice: "Incidente preso in carico"
|
||||
end
|
||||
|
||||
def resolve
|
||||
incident = Ops::Incident.find(params[:id])
|
||||
incident.resolve!
|
||||
redirect_to admin_ops_path, notice: "Incidente risolto"
|
||||
end
|
||||
|
||||
def mute
|
||||
incident = Ops::Incident.find(params[:id])
|
||||
incident.mute!(duration: 24.hours)
|
||||
redirect_to admin_ops_path, notice: "Notifiche sospese per 24 ore"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ops_summary
|
||||
{
|
||||
open_critical: Ops::Incident.critical_open.count,
|
||||
open_warning: Ops::Incident.open.where(severity: "warning").count,
|
||||
open_total: Ops::Incident.open.count
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
28
backend/app/controllers/ops/health_controller.rb
Normal file
28
backend/app/controllers/ops/health_controller.rb
Normal file
@@ -0,0 +1,28 @@
|
||||
module Ops
|
||||
class HealthController < ActionController::API
|
||||
before_action :authorize_deep_health!
|
||||
|
||||
def show
|
||||
summary = Ops::HealthChecks.new.summary
|
||||
status_code = case summary[:status]
|
||||
when "ok" then :ok
|
||||
when "degraded" then :service_unavailable
|
||||
else :service_unavailable
|
||||
end
|
||||
|
||||
render json: summary, status: status_code
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def authorize_deep_health!
|
||||
token = ENV["OPS_HEALTH_TOKEN"].presence
|
||||
return if token.blank?
|
||||
|
||||
provided = request.headers["X-Ops-Token"].presence || params[:token].presence
|
||||
return head :unauthorized if provided.blank?
|
||||
|
||||
head :unauthorized unless ActiveSupport::SecurityUtils.secure_compare(provided, token)
|
||||
end
|
||||
end
|
||||
end
|
||||
33
backend/app/jobs/ops/health_monitor_job.rb
Normal file
33
backend/app/jobs/ops/health_monitor_job.rb
Normal file
@@ -0,0 +1,33 @@
|
||||
module Ops
|
||||
class HealthMonitorJob
|
||||
include Sidekiq::Job
|
||||
|
||||
sidekiq_options retry: 1, queue: "default"
|
||||
|
||||
INTERVAL_SECS = ENV.fetch("OPS_HEALTH_INTERVAL_SECS", "180").to_i
|
||||
REDIS_CHAIN_KEY = "ops:health_monitor:chain"
|
||||
HEARTBEAT_KEY = "ops:health_monitor:last_run"
|
||||
|
||||
def self.ensure_chain
|
||||
return unless redis
|
||||
|
||||
return if redis.get(REDIS_CHAIN_KEY)
|
||||
|
||||
redis.set(REDIS_CHAIN_KEY, "1", ex: INTERVAL_SECS * 2)
|
||||
perform_in(INTERVAL_SECS)
|
||||
end
|
||||
|
||||
def self.redis
|
||||
@redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
rescue Redis::CannotConnectError
|
||||
nil
|
||||
end
|
||||
|
||||
def perform
|
||||
self.class.redis&.set(HEARTBEAT_KEY, Time.current.to_i, ex: INTERVAL_SECS * 3)
|
||||
Ops::HealthChecks.new.call
|
||||
ensure
|
||||
self.class.perform_in(INTERVAL_SECS)
|
||||
end
|
||||
end
|
||||
end
|
||||
11
backend/app/mailers/ops/alert_mailer.rb
Normal file
11
backend/app/mailers/ops/alert_mailer.rb
Normal file
@@ -0,0 +1,11 @@
|
||||
module Ops
|
||||
class AlertMailer < ApplicationMailer
|
||||
def incident(incident)
|
||||
@incident = incident
|
||||
mail(
|
||||
to: ENV.fetch("OPS_ALERT_EMAIL"),
|
||||
subject: "[Match Live TV Ops] #{incident.severity.upcase}: #{incident.title}"
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
43
backend/app/models/ops/incident.rb
Normal file
43
backend/app/models/ops/incident.rb
Normal file
@@ -0,0 +1,43 @@
|
||||
module Ops
|
||||
class Incident < ApplicationRecord
|
||||
self.table_name = "ops_incidents"
|
||||
|
||||
KINDS = %w[
|
||||
disk_space recordings_size service_down http_public sidekiq_stale sidekiq_dead
|
||||
log_pattern garage_storage
|
||||
].freeze
|
||||
SEVERITIES = %w[critical warning info].freeze
|
||||
STATUSES = %w[open acknowledged resolved].freeze
|
||||
|
||||
validates :kind, :severity, :status, :title, :fingerprint, presence: true
|
||||
validates :severity, inclusion: { in: SEVERITIES }
|
||||
validates :status, inclusion: { in: STATUSES }
|
||||
validates :occurrence_count, numericality: { greater_than: 0 }
|
||||
|
||||
scope :open, -> { where(status: %w[open acknowledged]) }
|
||||
scope :unresolved, -> { where.not(status: "resolved") }
|
||||
scope :critical_open, -> { open.where(severity: "critical") }
|
||||
scope :recent, -> { order(last_seen_at: :desc) }
|
||||
scope :since, ->(time) { where("last_seen_at >= ?", time) }
|
||||
|
||||
def open?
|
||||
status.in?(%w[open acknowledged])
|
||||
end
|
||||
|
||||
def muted?
|
||||
muted_until.present? && muted_until.future?
|
||||
end
|
||||
|
||||
def acknowledge!
|
||||
update!(status: "acknowledged", acknowledged_at: Time.current)
|
||||
end
|
||||
|
||||
def resolve!
|
||||
update!(status: "resolved")
|
||||
end
|
||||
|
||||
def mute!(duration: 24.hours)
|
||||
update!(muted_until: duration.from_now)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -84,26 +84,57 @@ module Admin
|
||||
def disk_usage_for(path)
|
||||
return unavailable_disk unless path.present? && (Dir.exist?(path) || File.exist?(path))
|
||||
|
||||
usage = disk_usage_via_statvfs(path) || disk_usage_via_df(path)
|
||||
return unavailable_disk unless usage
|
||||
|
||||
usage.merge(dir_size_bytes: recordings_dir_size(path))
|
||||
rescue StandardError
|
||||
unavailable_disk
|
||||
end
|
||||
|
||||
def disk_usage_via_statvfs(path)
|
||||
return nil unless File.respond_to?(:statvfs)
|
||||
|
||||
stat = File.statvfs(path)
|
||||
total = stat.blocks * stat.frsize
|
||||
free = stat.bavail * stat.frsize
|
||||
used = total - free
|
||||
build_disk_usage(path, total: total, used: used, free: free)
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
|
||||
def disk_usage_via_df(path)
|
||||
output = shell_out("df -B1 --output=size,used,avail,pcent,target #{Shellwords.escape(path)} 2>/dev/null").strip
|
||||
line = output.lines.map(&:strip).reject(&:blank?).last
|
||||
return nil if line.blank? || line.start_with?("Filesystem")
|
||||
|
||||
parts = line.split
|
||||
return nil if parts.size < 5
|
||||
|
||||
total = parts[0].to_i
|
||||
used = parts[1].to_i
|
||||
free = parts[2].to_i
|
||||
pct = parts[3].delete("%").to_f
|
||||
build_disk_usage(path, total: total, used: used, free: free, used_percent: pct)
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
|
||||
def build_disk_usage(path, total:, used:, free:, used_percent: nil)
|
||||
{
|
||||
path: path,
|
||||
total_bytes: total,
|
||||
used_bytes: used,
|
||||
free_bytes: free,
|
||||
used_percent: total.positive? ? ((used.to_f / total) * 100).round(1) : 0.0,
|
||||
dir_size_bytes: recordings_dir_size(path)
|
||||
used_percent: used_percent || (total.positive? ? ((used.to_f / total) * 100).round(1) : 0.0)
|
||||
}
|
||||
rescue StandardError
|
||||
unavailable_disk
|
||||
end
|
||||
|
||||
def recordings_dir_size(path)
|
||||
return nil unless path == RECORDINGS_PATH && Dir.exist?(path)
|
||||
|
||||
`du -sb #{Shellwords.escape(path)} 2>/dev/null`.split.first.to_i
|
||||
shell_out("du -sb #{Shellwords.escape(path)} 2>/dev/null").split.first.to_i
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
@@ -112,6 +143,10 @@ module Admin
|
||||
{ path: nil, total_bytes: nil, used_bytes: nil, free_bytes: nil, used_percent: nil, dir_size_bytes: nil }
|
||||
end
|
||||
|
||||
def shell_out(cmd)
|
||||
`#{cmd}`
|
||||
end
|
||||
|
||||
def read_network
|
||||
rx, tx = parse_net_bytes
|
||||
prev = MetricsStore.net_totals
|
||||
|
||||
248
backend/app/services/ops/health_checks.rb
Normal file
248
backend/app/services/ops/health_checks.rb
Normal file
@@ -0,0 +1,248 @@
|
||||
module Ops
|
||||
class HealthChecks
|
||||
Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true)
|
||||
|
||||
DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f }
|
||||
DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f }
|
||||
RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes }
|
||||
RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes }
|
||||
SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
|
||||
|
||||
def call
|
||||
findings = [
|
||||
check_disk_root,
|
||||
check_recordings_size,
|
||||
check_postgres,
|
||||
check_redis,
|
||||
check_mediamtx,
|
||||
check_garage,
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_public
|
||||
]
|
||||
|
||||
findings.each do |finding|
|
||||
if finding.healthy
|
||||
Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
|
||||
else
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: finding.kind,
|
||||
severity: finding.severity,
|
||||
title: finding.title,
|
||||
message: finding.message,
|
||||
metadata: finding.metadata,
|
||||
fingerprint: finding.fingerprint
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
findings
|
||||
end
|
||||
|
||||
def summary
|
||||
findings = [
|
||||
check_disk_root,
|
||||
check_recordings_size,
|
||||
check_postgres,
|
||||
check_redis,
|
||||
check_mediamtx,
|
||||
check_garage,
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_public
|
||||
]
|
||||
|
||||
status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
|
||||
"down"
|
||||
elsif findings.any? { |f| !f.healthy }
|
||||
"degraded"
|
||||
else
|
||||
"ok"
|
||||
end
|
||||
|
||||
{
|
||||
status: status,
|
||||
checked_at: Time.current.iso8601,
|
||||
checks: findings.map { |f| finding_to_hash(f) }
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def check_disk_root
|
||||
disk = Admin::HostMetrics.new.send(:read_disk)[:root]
|
||||
pct = disk[:used_percent].to_f
|
||||
healthy = pct < DISK_WARN_PERCENT.call
|
||||
severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "disk_space",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno",
|
||||
message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})",
|
||||
metadata: disk.transform_keys(&:to_s),
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
end
|
||||
|
||||
def check_recordings_size
|
||||
path = MatchLiveTv.recordings_local_path
|
||||
disk = Admin::HostMetrics.new.send(:disk_usage_for, path)
|
||||
size = disk[:dir_size_bytes].to_i
|
||||
healthy = size < RECORDINGS_WARN_BYTES.call
|
||||
severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "recordings_size",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato",
|
||||
message: "Contenuto #{path}: #{format_bytes(size)}",
|
||||
metadata: { "path" => path, "dir_size_bytes" => size },
|
||||
fingerprint: "recordings_size:local"
|
||||
)
|
||||
end
|
||||
|
||||
def check_postgres
|
||||
ActiveRecord::Base.connection.execute("SELECT 1")
|
||||
ok_finding("service_down:postgres", "PostgreSQL OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_redis
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
redis.ping
|
||||
ok_finding("service_down:redis", "Redis OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_mediamtx
|
||||
Mediamtx::Client.new.list_paths
|
||||
ok_finding("service_down:mediamtx", "MediaMTX OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_garage
|
||||
return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local?
|
||||
|
||||
require "aws-sdk-s3"
|
||||
client = Aws::S3::Client.new(
|
||||
endpoint: MatchLiveTv.replay_storage_endpoint,
|
||||
region: MatchLiveTv.replay_storage_region,
|
||||
access_key_id: MatchLiveTv.replay_storage_access_key_id,
|
||||
secret_access_key: MatchLiveTv.replay_storage_secret_access_key,
|
||||
force_path_style: MatchLiveTv.replay_storage_force_path_style?
|
||||
)
|
||||
client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket)
|
||||
ok_finding("garage_storage:ok", "Garage storage OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_sidekiq_heartbeat
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i
|
||||
age = last.positive? ? Time.current.to_i - last : nil
|
||||
healthy = age.present? && age < SIDEKIQ_STALE_SECS.call
|
||||
|
||||
Finding.new(
|
||||
kind: "sidekiq_stale",
|
||||
severity: "critical",
|
||||
healthy: healthy,
|
||||
title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde",
|
||||
message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)",
|
||||
metadata: { "last_heartbeat_age_secs" => age },
|
||||
fingerprint: "sidekiq_stale:heartbeat"
|
||||
)
|
||||
end
|
||||
|
||||
def check_sidekiq_dead
|
||||
dead_count = Sidekiq::DeadSet.new.size
|
||||
healthy = dead_count.zero?
|
||||
severity = dead_count > 5 ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "sidekiq_dead",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue",
|
||||
message: "#{dead_count} job nella dead queue",
|
||||
metadata: { "dead_count" => dead_count },
|
||||
fingerprint: "sidekiq_dead:count"
|
||||
)
|
||||
end
|
||||
|
||||
def check_http_public
|
||||
return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
|
||||
|
||||
url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
|
||||
response = Faraday.get(url) { |req| req.options.timeout = 10 }
|
||||
healthy = response.status == 200
|
||||
|
||||
Finding.new(
|
||||
kind: "http_public",
|
||||
severity: "critical",
|
||||
healthy: healthy,
|
||||
title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
|
||||
message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
|
||||
metadata: { "url" => url, "status" => response.status },
|
||||
fingerprint: "http_public:up"
|
||||
)
|
||||
rescue Faraday::Error => e
|
||||
fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def ok_finding(fingerprint, title)
|
||||
Finding.new(
|
||||
kind: fingerprint.split(":").first,
|
||||
severity: "info",
|
||||
healthy: true,
|
||||
title: title,
|
||||
message: title,
|
||||
metadata: {},
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
end
|
||||
|
||||
def fail_finding(kind, severity, fingerprint, title, message)
|
||||
Finding.new(
|
||||
kind: kind,
|
||||
severity: severity,
|
||||
healthy: false,
|
||||
title: title,
|
||||
message: message,
|
||||
metadata: {},
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
end
|
||||
|
||||
def finding_to_hash(finding)
|
||||
{
|
||||
kind: finding.kind,
|
||||
severity: finding.severity,
|
||||
healthy: finding.healthy,
|
||||
title: finding.title,
|
||||
message: finding.message,
|
||||
metadata: finding.metadata,
|
||||
fingerprint: finding.fingerprint
|
||||
}
|
||||
end
|
||||
|
||||
def format_bytes(bytes)
|
||||
return "—" if bytes.nil?
|
||||
|
||||
units = %w[B KB MB GB TB]
|
||||
size = bytes.to_f
|
||||
unit = units.shift
|
||||
while size >= 1024 && units.any?
|
||||
size /= 1024
|
||||
unit = units.shift
|
||||
end
|
||||
"#{size.round(size >= 10 ? 0 : 1)} #{unit}"
|
||||
end
|
||||
end
|
||||
end
|
||||
67
backend/app/services/ops/incident_recorder.rb
Normal file
67
backend/app/services/ops/incident_recorder.rb
Normal file
@@ -0,0 +1,67 @@
|
||||
module Ops
|
||||
class IncidentRecorder
|
||||
COOLDOWN = -> { ENV.fetch("OPS_NOTIFY_COOLDOWN_MINUTES", "30").to_i.minutes }
|
||||
|
||||
class << self
|
||||
def record(finding)
|
||||
new.record(finding)
|
||||
end
|
||||
|
||||
def resolve(fingerprint:)
|
||||
new.resolve(fingerprint: fingerprint)
|
||||
end
|
||||
end
|
||||
|
||||
def record(finding)
|
||||
fingerprint = finding[:fingerprint] || fingerprint_for(finding)
|
||||
incident = Ops::Incident.open.find_by(fingerprint: fingerprint)
|
||||
|
||||
if incident
|
||||
incident.occurrence_count += 1
|
||||
incident.last_seen_at = Time.current
|
||||
incident.title = finding[:title] if finding[:title].present?
|
||||
incident.message = finding[:message] if finding[:message].present?
|
||||
incident.metadata = (incident.metadata || {}).merge(finding[:metadata] || {})
|
||||
incident.severity = finding[:severity] if finding[:severity].present?
|
||||
incident.status = "open" if incident.status == "acknowledged"
|
||||
else
|
||||
now = Time.current
|
||||
incident = Ops::Incident.new(
|
||||
kind: finding[:kind],
|
||||
severity: finding[:severity],
|
||||
status: "open",
|
||||
title: finding[:title],
|
||||
message: finding[:message],
|
||||
metadata: finding[:metadata] || {},
|
||||
fingerprint: fingerprint,
|
||||
occurrence_count: 1,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now
|
||||
)
|
||||
end
|
||||
|
||||
incident.save!
|
||||
maybe_notify!(incident)
|
||||
incident
|
||||
end
|
||||
|
||||
def resolve(fingerprint:)
|
||||
Ops::Incident.open.where(fingerprint: fingerprint).find_each(&:resolve!)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fingerprint_for(finding)
|
||||
Digest::SHA256.hexdigest([finding[:kind], finding[:title]].join("|"))
|
||||
end
|
||||
|
||||
def maybe_notify!(incident)
|
||||
return if incident.muted?
|
||||
return unless Ops::Notifier.notify_severity?(incident.severity)
|
||||
return if incident.notified_at.present? && incident.notified_at > COOLDOWN.call.ago
|
||||
|
||||
Ops::Notifier.new.notify(incident)
|
||||
incident.update_column(:notified_at, Time.current)
|
||||
end
|
||||
end
|
||||
end
|
||||
37
backend/app/services/ops/log_pattern_matcher.rb
Normal file
37
backend/app/services/ops/log_pattern_matcher.rb
Normal file
@@ -0,0 +1,37 @@
|
||||
module Ops
|
||||
class LogPatternMatcher
|
||||
Match = Struct.new(:name, :severity, :line, keyword_init: true)
|
||||
|
||||
def initialize(patterns: nil)
|
||||
@patterns = patterns || load_patterns
|
||||
end
|
||||
|
||||
def scan(text)
|
||||
return [] if text.blank?
|
||||
|
||||
matches = []
|
||||
text.each_line do |line|
|
||||
@patterns.each do |pattern|
|
||||
next unless line.match?(pattern[:regex])
|
||||
|
||||
matches << Match.new(name: pattern[:name], severity: pattern[:severity], line: line.strip)
|
||||
end
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def load_patterns
|
||||
path = Rails.root.join("config/ops_log_patterns.yml")
|
||||
raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: true) || {}
|
||||
Array(raw["patterns"]).map do |entry|
|
||||
{
|
||||
name: entry["name"].to_s,
|
||||
severity: entry["severity"].to_s,
|
||||
regex: Regexp.new(entry["regex"], Regexp::IGNORECASE)
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
26
backend/app/services/ops/log_scanner.rb
Normal file
26
backend/app/services/ops/log_scanner.rb
Normal file
@@ -0,0 +1,26 @@
|
||||
module Ops
|
||||
class LogScanner
|
||||
def initialize(matcher: LogPatternMatcher.new)
|
||||
@matcher = matcher
|
||||
end
|
||||
|
||||
def ingest(text)
|
||||
matches = @matcher.scan(text)
|
||||
matches.each { |match| record_match(match) }
|
||||
matches.size
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def record_match(match)
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: "log_pattern",
|
||||
severity: match.severity,
|
||||
title: "Log: #{match.name}",
|
||||
message: match.line.truncate(500),
|
||||
metadata: { "pattern" => match.name, "sample" => match.line },
|
||||
fingerprint: "log_pattern:#{match.name}"
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
65
backend/app/services/ops/notifier.rb
Normal file
65
backend/app/services/ops/notifier.rb
Normal file
@@ -0,0 +1,65 @@
|
||||
module Ops
|
||||
class Notifier
|
||||
class Error < StandardError; end
|
||||
|
||||
def self.notify_severity?(severity)
|
||||
levels = ENV.fetch("OPS_NOTIFY_SEVERITIES", "critical").split(",").map(&:strip)
|
||||
levels.include?(severity.to_s)
|
||||
end
|
||||
|
||||
def notify(incident)
|
||||
notify_ntfy(incident) if ntfy_url.present?
|
||||
notify_email(incident) if alert_email.present?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def notify_ntfy(incident)
|
||||
priority = incident.severity == "critical" ? "5" : "3"
|
||||
tags = incident.severity == "critical" ? "rotating_light" : "warning"
|
||||
|
||||
conn = Faraday.new do |f|
|
||||
f.adapter Faraday.default_adapter
|
||||
f.options.timeout = 10
|
||||
f.options.open_timeout = 5
|
||||
end
|
||||
|
||||
headers = {
|
||||
"Title" => "[Match Live TV] #{incident.title}",
|
||||
"Priority" => priority,
|
||||
"Tags" => tags
|
||||
}
|
||||
headers["Authorization"] = "Bearer #{ntfy_token}" if ntfy_token.present?
|
||||
|
||||
body = incident.message.to_s
|
||||
body += "\n\n(#{incident.occurrence_count} occorrenze)" if incident.occurrence_count > 1
|
||||
|
||||
response = conn.post(ntfy_url, body, headers)
|
||||
return if response.success?
|
||||
|
||||
Rails.logger.warn("[Ops::Notifier] ntfy failed: #{response.status} #{response.body}")
|
||||
rescue Faraday::Error => e
|
||||
Rails.logger.warn("[Ops::Notifier] ntfy error: #{e.message}")
|
||||
end
|
||||
|
||||
def notify_email(incident)
|
||||
return unless defined?(ActionMailer)
|
||||
|
||||
Ops::AlertMailer.incident(incident).deliver_later
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn("[Ops::Notifier] email error: #{e.message}")
|
||||
end
|
||||
|
||||
def ntfy_url
|
||||
ENV["OPS_NTFY_URL"].presence
|
||||
end
|
||||
|
||||
def ntfy_token
|
||||
ENV["OPS_NTFY_TOKEN"].presence
|
||||
end
|
||||
|
||||
def alert_email
|
||||
ENV["OPS_ALERT_EMAIL"].presence
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -54,6 +54,19 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" style="margin-bottom:1.25rem">
|
||||
<h2>Stato piattaforma (Ops)</h2>
|
||||
<% critical = Ops::Incident.critical_open.count %>
|
||||
<% warnings = Ops::Incident.open.where(severity: "warning").count %>
|
||||
<% if critical.positive? %>
|
||||
<p class="admin-flash" style="margin-bottom:0.75rem"><%= critical %> incidente/i critico/i aperto/i — <%= link_to "Vedi dashboard Ops", admin_ops_path %></p>
|
||||
<% elsif warnings.positive? %>
|
||||
<p class="kpi-sub" style="margin-bottom:0.75rem"><%= warnings %> warning — <%= link_to "Dashboard Ops", admin_ops_path %></p>
|
||||
<% else %>
|
||||
<p class="empty" style="margin:0">Nessun incidente aperto. <%= link_to "Dashboard Ops", admin_ops_path %></p>
|
||||
<% end %>
|
||||
</section>
|
||||
|
||||
<section class="charts-grid" style="grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));">
|
||||
<div class="chart-card">
|
||||
<h3>Disco sistema</h3>
|
||||
|
||||
83
backend/app/views/admin/ops/index.html.erb
Normal file
83
backend/app/views/admin/ops/index.html.erb
Normal file
@@ -0,0 +1,83 @@
|
||||
<% content_for :body_class, "admin-body" %>
|
||||
|
||||
<section class="kpi-grid">
|
||||
<div class="kpi-card <%= @summary[:open_critical].positive? ? 'kpi-card--danger' : 'kpi-card--accent' %>">
|
||||
<div class="kpi-label">Critici aperti</div>
|
||||
<div class="kpi-value"><%= @summary[:open_critical] %></div>
|
||||
</div>
|
||||
<div class="kpi-card">
|
||||
<div class="kpi-label">Warning aperti</div>
|
||||
<div class="kpi-value"><%= @summary[:open_warning] %></div>
|
||||
</div>
|
||||
<div class="kpi-card">
|
||||
<div class="kpi-label">Totale aperti</div>
|
||||
<div class="kpi-value"><%= @summary[:open_total] %></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="panel" style="margin-bottom:1.5rem">
|
||||
<h2>Incidenti aperti</h2>
|
||||
<% if @open_incidents.any? %>
|
||||
<table class="admin-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Severità</th>
|
||||
<th>Tipo</th>
|
||||
<th>Titolo</th>
|
||||
<th>Occ.</th>
|
||||
<th>Ultimo</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<% @open_incidents.each do |inc| %>
|
||||
<tr>
|
||||
<td><span class="badge badge--<%= inc.severity == 'critical' ? 'live' : 'paused' %>"><%= inc.severity %></span></td>
|
||||
<td class="muted"><%= inc.kind %></td>
|
||||
<td>
|
||||
<strong><%= inc.title %></strong>
|
||||
<% if inc.message.present? %>
|
||||
<div class="muted" style="font-size:0.85rem;margin-top:0.25rem"><%= truncate(inc.message, length: 120) %></div>
|
||||
<% end %>
|
||||
</td>
|
||||
<td><%= inc.occurrence_count %></td>
|
||||
<td class="muted"><%= inc.last_seen_at&.strftime("%d/%m %H:%M") %></td>
|
||||
<td class="admin-actions">
|
||||
<% unless inc.status == 'acknowledged' %>
|
||||
<%= button_to "Preso in carico", acknowledge_admin_op_path(inc), method: :post, class: "admin-btn admin-btn--sm" %>
|
||||
<% end %>
|
||||
<%= button_to "Risolvi", resolve_admin_op_path(inc), method: :post, class: "admin-btn admin-btn--sm" %>
|
||||
<%= button_to "Mute 24h", mute_admin_op_path(inc), method: :post, class: "admin-btn admin-btn--sm admin-btn--outline" %>
|
||||
</td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</tbody>
|
||||
</table>
|
||||
<% else %>
|
||||
<p class="empty">Nessun incidente aperto. La piattaforma risulta sana.</p>
|
||||
<% end %>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<h2>Risolti (ultimi 30 giorni)</h2>
|
||||
<% if @resolved_incidents.any? %>
|
||||
<table class="admin-table">
|
||||
<thead>
|
||||
<tr><th>Severità</th><th>Tipo</th><th>Titolo</th><th>Occ.</th><th>Risolto</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<% @resolved_incidents.each do |inc| %>
|
||||
<tr>
|
||||
<td><span class="badge badge--connecting"><%= inc.severity %></span></td>
|
||||
<td class="muted"><%= inc.kind %></td>
|
||||
<td><%= inc.title %></td>
|
||||
<td><%= inc.occurrence_count %></td>
|
||||
<td class="muted"><%= inc.updated_at.strftime("%d/%m %H:%M") %></td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</tbody>
|
||||
</table>
|
||||
<% else %>
|
||||
<p class="empty">Nessun incidente risolto di recente.</p>
|
||||
<% end %>
|
||||
</div>
|
||||
@@ -16,6 +16,10 @@
|
||||
<nav class="admin-nav">
|
||||
<% if admin_logged_in? %>
|
||||
<%= link_to "Dashboard", admin_root_path, class: ("active" if controller_name == "dashboard") %>
|
||||
<% ops_critical = Ops::Incident.critical_open.count %>
|
||||
<%= link_to admin_ops_path, class: ("active" if controller_name == "ops") do %>
|
||||
Ops<% if ops_critical.positive? %> <span class="admin-nav-badge"><%= ops_critical %></span><% end %>
|
||||
<% end %>
|
||||
<%= link_to "Società e squadre", admin_clubs_path, class: ("active" if controller_name.in?(%w[clubs teams])) %>
|
||||
<%= link_to "Fatturazione", admin_billing_path, class: ("active" if controller_name.in?(%w[billing billing_invoices])) %>
|
||||
<%= link_to "YouTube", admin_youtube_platform_path, class: ("active" if controller_name == "youtube") %>
|
||||
|
||||
10
backend/app/views/ops/alert_mailer/incident.text.erb
Normal file
10
backend/app/views/ops/alert_mailer/incident.text.erb
Normal file
@@ -0,0 +1,10 @@
|
||||
<%= @incident.title %>
|
||||
|
||||
Severità: <%= @incident.severity %>
|
||||
Tipo: <%= @incident.kind %>
|
||||
Occorrenze: <%= @incident.occurrence_count %>
|
||||
Ultimo avvistamento: <%= @incident.last_seen_at %>
|
||||
|
||||
<%= @incident.message %>
|
||||
|
||||
Dashboard: <%= ENV.fetch("APP_PUBLIC_URL", "https://www.matchlivetv.it").chomp("/") %>/admin/ops
|
||||
@@ -13,7 +13,7 @@ Rails.application.configure do
|
||||
config.ssl_options = {
|
||||
redirect: {
|
||||
exclude: ->(request) {
|
||||
request.path == "/up" || request.path == "/cable" || request.path.start_with?("/hls/")
|
||||
request.path == "/up" || request.path == "/up/deep" || request.path == "/cable" || request.path.start_with?("/hls/")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ Rails.application.configure do
|
||||
|
||||
config.host_authorization = {
|
||||
exclude: ->(request) {
|
||||
request.path == "/up" || request.path.start_with?("/cable") || request.path.start_with?("/hls/")
|
||||
request.path == "/up" || request.path == "/up/deep" || request.path.start_with?("/cable") || request.path.start_with?("/hls/")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
21
backend/config/initializers/ops_log_subscriber.rb
Normal file
21
backend/config/initializers/ops_log_subscriber.rb
Normal file
@@ -0,0 +1,21 @@
|
||||
if Rails.env.production? && ENV["OPS_LOG_SUBSCRIBER"] == "true"
|
||||
ActiveSupport::Notifications.subscribe("process_action.action_controller") do |*args|
|
||||
event = ActiveSupport::Notifications::Event.new(*args)
|
||||
next unless event.payload[:status].to_i >= 500
|
||||
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: "log_pattern",
|
||||
severity: "warning",
|
||||
title: "HTTP 500 in Rails",
|
||||
message: "#{event.payload[:method]} #{event.payload[:path]} → #{event.payload[:status]}",
|
||||
metadata: {
|
||||
"controller" => event.payload[:controller],
|
||||
"action" => event.payload[:action],
|
||||
"status" => event.payload[:status]
|
||||
},
|
||||
fingerprint: "rails_500:#{event.payload[:controller]}##{event.payload[:action]}"
|
||||
)
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn("[Ops::LogSubscriber] #{e.message}")
|
||||
end
|
||||
end
|
||||
33
backend/config/initializers/sentry.rb
Normal file
33
backend/config/initializers/sentry.rb
Normal file
@@ -0,0 +1,33 @@
|
||||
if ENV["SENTRY_DSN"].present? && !Rails.env.test?
|
||||
begin
|
||||
require "sentry-ruby"
|
||||
require "sentry-rails"
|
||||
|
||||
Sentry.init do |config|
|
||||
config.dsn = ENV["SENTRY_DSN"]
|
||||
config.breadcrumbs_logger = %i[active_support_logger http_logger]
|
||||
config.traces_sample_rate = ENV.fetch("SENTRY_TRACES_SAMPLE_RATE", "0.1").to_f
|
||||
config.environment = Rails.env
|
||||
|
||||
config.before_send = lambda do |event, _hint|
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: "log_pattern",
|
||||
severity: "warning",
|
||||
title: "Sentry: #{event.message}",
|
||||
message: event.message,
|
||||
metadata: {
|
||||
"sentry_event_id" => event.event_id,
|
||||
"level" => event.level
|
||||
},
|
||||
fingerprint: "sentry:#{event.fingerprint&.first || event.event_id}"
|
||||
)
|
||||
event
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn("[Sentry] ops incident hook: #{e.message}")
|
||||
event
|
||||
end
|
||||
end
|
||||
rescue LoadError
|
||||
Rails.logger.warn("[Sentry] gem non installata — imposta SENTRY_DSN dopo bundle add sentry-ruby sentry-rails")
|
||||
end
|
||||
end
|
||||
@@ -5,6 +5,7 @@ Sidekiq.configure_server do |config|
|
||||
|
||||
config.on(:startup) do
|
||||
StreamPublisherSyncJob.ensure_chain
|
||||
Ops::HealthMonitorJob.ensure_chain
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
16
backend/config/ops_log_patterns.yml
Normal file
16
backend/config/ops_log_patterns.yml
Normal file
@@ -0,0 +1,16 @@
|
||||
patterns:
|
||||
- name: http_500
|
||||
severity: warning
|
||||
regex: 'Completed 500'
|
||||
- name: disk_full
|
||||
severity: critical
|
||||
regex: 'No space left on device'
|
||||
- name: db_connection
|
||||
severity: critical
|
||||
regex: 'PG::(ConnectionBad|UnableToSend)'
|
||||
- name: sidekiq_failure
|
||||
severity: warning
|
||||
regex: 'WARN.*fail'
|
||||
- name: fatal_error
|
||||
severity: critical
|
||||
regex: '\bFATAL\b'
|
||||
@@ -2,6 +2,7 @@ Rails.application.routes.draw do
|
||||
mount ActionCable.server => "/cable"
|
||||
|
||||
get "up" => "rails/health#show", as: :rails_health_check
|
||||
get "up/deep" => "ops/health#show", as: :ops_deep_health
|
||||
|
||||
namespace :api do
|
||||
namespace :v1 do
|
||||
@@ -76,6 +77,13 @@ Rails.application.routes.draw do
|
||||
|
||||
root to: "dashboard#index"
|
||||
get "metrics", to: "dashboard#metrics"
|
||||
resources :ops, only: [:index], controller: "ops" do
|
||||
member do
|
||||
post :acknowledge
|
||||
post :resolve
|
||||
post :mute
|
||||
end
|
||||
end
|
||||
get "billing", to: "billing#index", as: :billing
|
||||
post "billing/payments/:payment_id/attach_pdf", to: "billing#attach_pdf", as: :billing_payment_attach_pdf
|
||||
resources :teams, only: %i[show]
|
||||
|
||||
26
backend/db/migrate/20260611120000_create_ops_incidents.rb
Normal file
26
backend/db/migrate/20260611120000_create_ops_incidents.rb
Normal file
@@ -0,0 +1,26 @@
|
||||
class CreateOpsIncidents < ActiveRecord::Migration[7.2]
|
||||
def change
|
||||
create_table :ops_incidents, id: :uuid do |t|
|
||||
t.string :kind, null: false
|
||||
t.string :severity, null: false, default: "warning"
|
||||
t.string :status, null: false, default: "open"
|
||||
t.string :title, null: false
|
||||
t.text :message
|
||||
t.jsonb :metadata, null: false, default: {}
|
||||
t.string :fingerprint, null: false
|
||||
t.integer :occurrence_count, null: false, default: 1
|
||||
t.datetime :first_seen_at, null: false
|
||||
t.datetime :last_seen_at, null: false
|
||||
t.datetime :notified_at
|
||||
t.datetime :acknowledged_at
|
||||
t.datetime :muted_until
|
||||
|
||||
t.timestamps
|
||||
end
|
||||
|
||||
add_index :ops_incidents, :fingerprint
|
||||
add_index :ops_incidents, %i[status severity]
|
||||
add_index :ops_incidents, :last_seen_at
|
||||
add_index :ops_incidents, %i[fingerprint status]
|
||||
end
|
||||
end
|
||||
24
backend/db/schema.rb
generated
24
backend/db/schema.rb
generated
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.2].define(version: 2026_06_09_100000) do
|
||||
ActiveRecord::Schema[7.2].define(version: 2026_06_11_120000) do
|
||||
# These are extensions that must be enabled in order to support this database
|
||||
enable_extension "pgcrypto"
|
||||
enable_extension "plpgsql"
|
||||
@@ -160,6 +160,28 @@ ActiveRecord::Schema[7.2].define(version: 2026_06_09_100000) do
|
||||
t.index ["team_id"], name: "index_matches_on_team_id"
|
||||
end
|
||||
|
||||
create_table "ops_incidents", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
|
||||
t.string "kind", null: false
|
||||
t.string "severity", default: "warning", null: false
|
||||
t.string "status", default: "open", null: false
|
||||
t.string "title", null: false
|
||||
t.text "message"
|
||||
t.jsonb "metadata", default: {}, null: false
|
||||
t.string "fingerprint", null: false
|
||||
t.integer "occurrence_count", default: 1, null: false
|
||||
t.datetime "first_seen_at", null: false
|
||||
t.datetime "last_seen_at", null: false
|
||||
t.datetime "notified_at"
|
||||
t.datetime "acknowledged_at"
|
||||
t.datetime "muted_until"
|
||||
t.datetime "created_at", null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.index ["fingerprint", "status"], name: "index_ops_incidents_on_fingerprint_and_status"
|
||||
t.index ["fingerprint"], name: "index_ops_incidents_on_fingerprint"
|
||||
t.index ["last_seen_at"], name: "index_ops_incidents_on_last_seen_at"
|
||||
t.index ["status", "severity"], name: "index_ops_incidents_on_status_and_severity"
|
||||
end
|
||||
|
||||
create_table "plans", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
|
||||
t.string "slug", null: false
|
||||
t.string "name", null: false
|
||||
|
||||
41
backend/lib/tasks/ops.rake
Normal file
41
backend/lib/tasks/ops.rake
Normal file
@@ -0,0 +1,41 @@
|
||||
namespace :ops do
|
||||
desc "Esegue health check ops e registra incidenti"
|
||||
task health_check: :environment do
|
||||
findings = Ops::HealthChecks.new.call
|
||||
unhealthy = findings.reject(&:healthy)
|
||||
puts "Health check completato: #{findings.size} controlli, #{unhealthy.size} problemi"
|
||||
unhealthy.each do |f|
|
||||
puts " [#{f.severity}] #{f.title}: #{f.message}"
|
||||
end
|
||||
end
|
||||
|
||||
desc "Ingestione log da stdin o file (rails ops:ingest_logs[path])"
|
||||
task :ingest_logs, [:path] => :environment do |_t, args|
|
||||
text = if args[:path].present?
|
||||
File.read(args[:path])
|
||||
else
|
||||
$stdin.read
|
||||
end
|
||||
count = Ops::LogScanner.new.ingest(text)
|
||||
puts "Log scanner: #{count} match trovati"
|
||||
end
|
||||
|
||||
desc "Invia notifica di test (ntfy/email)"
|
||||
task test_notify: :environment do
|
||||
incident = Ops::Incident.create!(
|
||||
kind: "test",
|
||||
severity: "critical",
|
||||
status: "open",
|
||||
title: "Test notifica ops",
|
||||
message: "Verifica canale notifiche Match Live TV — #{Time.current}",
|
||||
metadata: {},
|
||||
fingerprint: "test:#{SecureRandom.hex(4)}",
|
||||
occurrence_count: 1,
|
||||
first_seen_at: Time.current,
|
||||
last_seen_at: Time.current
|
||||
)
|
||||
Ops::Notifier.new.notify(incident)
|
||||
incident.resolve!
|
||||
puts "Notifica di test inviata"
|
||||
end
|
||||
end
|
||||
@@ -107,6 +107,25 @@ body.admin-body {
|
||||
background: linear-gradient(145deg, #1a1214 0%, var(--card) 60%);
|
||||
}
|
||||
|
||||
.kpi-card--danger {
|
||||
border-color: rgba(229, 57, 53, 0.75);
|
||||
background: linear-gradient(145deg, #2a1010 0%, var(--card) 55%);
|
||||
}
|
||||
|
||||
.admin-nav-badge {
|
||||
display: inline-block;
|
||||
min-width: 1.1rem;
|
||||
padding: 0 0.35rem;
|
||||
margin-left: 0.25rem;
|
||||
border-radius: 999px;
|
||||
background: var(--accent);
|
||||
color: #fff;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
line-height: 1.2rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.kpi-label {
|
||||
font-size: 0.75rem;
|
||||
color: var(--muted);
|
||||
|
||||
3
backend/spec/fixtures/ops/sample.log
vendored
Normal file
3
backend/spec/fixtures/ops/sample.log
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
I, [2026-06-10T19:45:10.123456 #1] INFO -- : [abc] Completed 500 Internal Server Error in 3ms
|
||||
E, [2026-06-10T19:45:11.123456 #1] ERROR -- : PG::ConnectionBad: could not connect to server
|
||||
E, [2026-06-10T19:45:12.123456 #1] ERROR -- : No space left on device @ dir_s_mkdir - /recordings
|
||||
@@ -10,7 +10,7 @@ RSpec.describe Billing::Invoice do
|
||||
amount_cents: 4000,
|
||||
status: "issued"
|
||||
)
|
||||
expect(invoice).not_to be_valid
|
||||
expect(invoice.valid?(:issue)).to be(false)
|
||||
expect(invoice.errors[:pdf]).to be_present
|
||||
end
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ require_relative "../config/environment"
|
||||
require "rspec/rails"
|
||||
|
||||
RSpec.configure do |config|
|
||||
config.include ActiveSupport::Testing::TimeHelpers
|
||||
|
||||
config.use_transactional_fixtures = true
|
||||
config.infer_spec_type_from_file_location!
|
||||
config.filter_rails_from_backtrace!
|
||||
|
||||
32
backend/spec/requests/ops/health_spec.rb
Normal file
32
backend/spec/requests/ops/health_spec.rb
Normal file
@@ -0,0 +1,32 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe "Ops deep health", type: :request do
|
||||
describe "GET /up/deep" do
|
||||
it "restituisce summary JSON" do
|
||||
allow_any_instance_of(Ops::HealthChecks).to receive(:summary).and_return(
|
||||
status: "ok",
|
||||
checked_at: Time.current.iso8601,
|
||||
checks: []
|
||||
)
|
||||
|
||||
get "/up/deep"
|
||||
|
||||
expect(response).to have_http_status(:ok)
|
||||
expect(JSON.parse(response.body)["status"]).to eq("ok")
|
||||
end
|
||||
|
||||
it "richiede token quando OPS_HEALTH_TOKEN è impostato" do
|
||||
allow(ENV).to receive(:[]).and_call_original
|
||||
allow(ENV).to receive(:[]).with("OPS_HEALTH_TOKEN").and_return("secret-token")
|
||||
allow_any_instance_of(Ops::HealthChecks).to receive(:summary).and_return(
|
||||
status: "ok", checked_at: Time.current.iso8601, checks: []
|
||||
)
|
||||
|
||||
get "/up/deep"
|
||||
expect(response).to have_http_status(:unauthorized)
|
||||
|
||||
get "/up/deep", headers: { "X-Ops-Token" => "secret-token" }
|
||||
expect(response).to have_http_status(:ok)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -26,6 +26,15 @@ RSpec.describe "Public regia", type: :request do
|
||||
end
|
||||
|
||||
it "mette in pausa e riprende la diretta" do
|
||||
mtx = instance_double(
|
||||
Mediamtx::Client,
|
||||
set_always_available: true,
|
||||
set_path_recording: true,
|
||||
delete_path: true,
|
||||
list_paths: []
|
||||
)
|
||||
allow(Mediamtx::Client).to receive(:new).and_return(mtx)
|
||||
|
||||
token = Sessions::RegiaAccess.new(session).issue_token!
|
||||
post public_regia_pause_path(token)
|
||||
expect(response).to have_http_status(:ok)
|
||||
|
||||
39
backend/spec/services/admin/host_metrics_spec.rb
Normal file
39
backend/spec/services/admin/host_metrics_spec.rb
Normal file
@@ -0,0 +1,39 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Admin::HostMetrics do
|
||||
subject(:metrics) { described_class.new }
|
||||
|
||||
describe "#build_disk_usage" do
|
||||
it "costruisce la struttura disco" do
|
||||
result = metrics.send(:build_disk_usage, "/", total: 115_964_116_992, used: 9_388_032_000, free: 106_576_084_992, used_percent: 9.0)
|
||||
|
||||
expect(result[:path]).to eq("/")
|
||||
expect(result[:total_bytes]).to eq(115_964_116_992)
|
||||
expect(result[:used_bytes]).to eq(9_388_032_000)
|
||||
expect(result[:free_bytes]).to eq(106_576_084_992)
|
||||
expect(result[:used_percent]).to eq(9.0)
|
||||
end
|
||||
end
|
||||
|
||||
describe "#disk_usage_for" do
|
||||
it "usa df quando statvfs non è disponibile" do
|
||||
allow(File).to receive(:respond_to?).and_call_original
|
||||
allow(File).to receive(:respond_to?).with(:statvfs).and_return(false)
|
||||
allow(Dir).to receive(:exist?).with("/recordings").and_return(true)
|
||||
allow(metrics).to receive(:disk_usage_via_statvfs).with("/recordings").and_return(nil)
|
||||
allow(metrics).to receive(:disk_usage_via_df).with("/recordings").and_return(
|
||||
path: "/recordings",
|
||||
total_bytes: 100_000,
|
||||
used_bytes: 9_000,
|
||||
free_bytes: 91_000,
|
||||
used_percent: 9.0
|
||||
)
|
||||
allow(metrics).to receive(:shell_out).with(/du -sb/).and_return("1048576\t/recordings\n")
|
||||
|
||||
result = metrics.send(:disk_usage_for, "/recordings")
|
||||
|
||||
expect(result[:total_bytes]).to eq(100_000)
|
||||
expect(result[:dir_size_bytes]).to eq(1_048_576)
|
||||
end
|
||||
end
|
||||
end
|
||||
69
backend/spec/services/ops/health_checks_spec.rb
Normal file
69
backend/spec/services/ops/health_checks_spec.rb
Normal file
@@ -0,0 +1,69 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::HealthChecks do
|
||||
describe "#call" do
|
||||
it "risolve incidenti quando il disco è sano" do
|
||||
Ops::Incident.create!(
|
||||
kind: "disk_space",
|
||||
severity: "warning",
|
||||
status: "open",
|
||||
title: "Disco",
|
||||
message: "alto",
|
||||
metadata: {},
|
||||
fingerprint: "disk_space:root",
|
||||
occurrence_count: 1,
|
||||
first_seen_at: Time.current,
|
||||
last_seen_at: Time.current
|
||||
)
|
||||
|
||||
allow_any_instance_of(described_class).to receive(:check_disk_root).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "disk_space", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "disk_space:root"
|
||||
)
|
||||
)
|
||||
%i[
|
||||
check_recordings_size check_postgres check_redis check_mediamtx check_garage
|
||||
check_sidekiq_heartbeat check_sidekiq_dead check_http_public
|
||||
].each do |method|
|
||||
allow_any_instance_of(described_class).to receive(method).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "test", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "#{method}:ok"
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
described_class.new.call
|
||||
|
||||
expect(Ops::Incident.find_by(fingerprint: "disk_space:root").status).to eq("resolved")
|
||||
end
|
||||
end
|
||||
|
||||
describe "#summary" do
|
||||
it "restituisce status ok quando tutti i check sono sani" do
|
||||
allow_any_instance_of(described_class).to receive(:check_disk_root).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "disk_space", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "disk_space:root"
|
||||
)
|
||||
)
|
||||
%i[
|
||||
check_recordings_size check_postgres check_redis check_mediamtx check_garage
|
||||
check_sidekiq_heartbeat check_sidekiq_dead check_http_public
|
||||
].each do |method|
|
||||
allow_any_instance_of(described_class).to receive(method).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "test", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "#{method}:ok"
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
summary = described_class.new.summary
|
||||
|
||||
expect(summary[:status]).to eq("ok")
|
||||
expect(summary[:checks].size).to eq(9)
|
||||
end
|
||||
end
|
||||
end
|
||||
55
backend/spec/services/ops/incident_recorder_spec.rb
Normal file
55
backend/spec/services/ops/incident_recorder_spec.rb
Normal file
@@ -0,0 +1,55 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::IncidentRecorder do
|
||||
describe ".record" do
|
||||
it "crea un incidente nuovo" do
|
||||
incident = described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "critical",
|
||||
title: "Disco pieno",
|
||||
message: "90% usato",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
|
||||
expect(incident).to be_persisted
|
||||
expect(incident.occurrence_count).to eq(1)
|
||||
expect(incident.status).to eq("open")
|
||||
end
|
||||
|
||||
it "deduplica per fingerprint incrementando occorrenze" do
|
||||
described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "critical",
|
||||
title: "Disco pieno",
|
||||
message: "90%",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
second = described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "critical",
|
||||
title: "Disco pieno",
|
||||
message: "91%",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
|
||||
expect(Ops::Incident.where(fingerprint: "disk_space:root").count).to eq(1)
|
||||
expect(second.occurrence_count).to eq(2)
|
||||
end
|
||||
end
|
||||
|
||||
describe ".resolve" do
|
||||
it "chiude incidenti aperti con la stessa fingerprint" do
|
||||
incident = described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "warning",
|
||||
title: "Test",
|
||||
message: "msg",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
|
||||
described_class.resolve(fingerprint: "disk_space:root")
|
||||
|
||||
expect(incident.reload.status).to eq("resolved")
|
||||
end
|
||||
end
|
||||
end
|
||||
24
backend/spec/services/ops/log_pattern_matcher_spec.rb
Normal file
24
backend/spec/services/ops/log_pattern_matcher_spec.rb
Normal file
@@ -0,0 +1,24 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::LogPatternMatcher do
|
||||
subject(:matcher) do
|
||||
described_class.new(
|
||||
patterns: [
|
||||
{ name: "http_500", severity: "warning", regex: /Completed 500/i },
|
||||
{ name: "disk_full", severity: "critical", regex: /No space left on device/i }
|
||||
]
|
||||
)
|
||||
end
|
||||
|
||||
it "trova pattern nei log" do
|
||||
log = <<~LOG
|
||||
I, [2026-06-10T19:45:10] Completed 500 Internal Server Error
|
||||
E, [2026-06-10T19:45:11] No space left on device - /recordings
|
||||
LOG
|
||||
|
||||
matches = matcher.scan(log)
|
||||
|
||||
expect(matches.map(&:name)).to contain_exactly("http_500", "disk_full")
|
||||
expect(matches.find { |m| m.name == "disk_full" }.severity).to eq("critical")
|
||||
end
|
||||
end
|
||||
9
backend/spec/services/ops/log_scanner_spec.rb
Normal file
9
backend/spec/services/ops/log_scanner_spec.rb
Normal file
@@ -0,0 +1,9 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::LogScanner do
|
||||
it "crea incidenti dai pattern trovati" do
|
||||
log = Rails.root.join("spec/fixtures/ops/sample.log").read
|
||||
|
||||
expect { described_class.new.ingest(log) }.to change(Ops::Incident, :count).by_at_least(2)
|
||||
end
|
||||
end
|
||||
43
backend/spec/services/ops/notifier_spec.rb
Normal file
43
backend/spec/services/ops/notifier_spec.rb
Normal file
@@ -0,0 +1,43 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::Notifier do
|
||||
let(:incident) do
|
||||
Ops::Incident.create!(
|
||||
kind: "test",
|
||||
severity: "critical",
|
||||
status: "open",
|
||||
title: "Test alert",
|
||||
message: "Messaggio di prova",
|
||||
metadata: {},
|
||||
fingerprint: "test:#{SecureRandom.hex(4)}",
|
||||
occurrence_count: 1,
|
||||
first_seen_at: Time.current,
|
||||
last_seen_at: Time.current
|
||||
)
|
||||
end
|
||||
|
||||
describe "#notify" do
|
||||
it "invia POST a ntfy quando configurato" do
|
||||
conn = instance_double(Faraday::Connection)
|
||||
response = instance_double(Faraday::Response, success?: true, status: 200, body: "")
|
||||
allow(Faraday).to receive(:new).and_return(conn)
|
||||
allow(conn).to receive(:post).and_return(response)
|
||||
allow(ENV).to receive(:[]).and_call_original
|
||||
allow(ENV).to receive(:[]).with("OPS_NTFY_URL").and_return("https://ntfy.example/ops")
|
||||
|
||||
described_class.new.notify(incident)
|
||||
|
||||
expect(conn).to have_received(:post).with("https://ntfy.example/ops", anything, hash_including("Title"))
|
||||
end
|
||||
end
|
||||
|
||||
describe ".notify_severity?" do
|
||||
it "rispetta OPS_NOTIFY_SEVERITIES" do
|
||||
allow(ENV).to receive(:fetch).and_call_original
|
||||
allow(ENV).to receive(:fetch).with("OPS_NOTIFY_SEVERITIES", "critical").and_return("critical,warning")
|
||||
|
||||
expect(described_class.notify_severity?("critical")).to be(true)
|
||||
expect(described_class.notify_severity?("info")).to be(false)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -4,10 +4,10 @@ RSpec.describe Youtube::BroadcastService do
|
||||
describe "#normalize_scheduled_start_time" do
|
||||
subject(:service) { described_class.allocate }
|
||||
|
||||
it "usa tra ~90 secondi se scheduled_at è assente" do
|
||||
it "usa tra ~20 secondi se scheduled_at è assente" do
|
||||
travel_to Time.zone.parse("2026-06-03 20:00:00 UTC") do
|
||||
t = service.send(:normalize_scheduled_start_time, nil)
|
||||
expect(t).to eq(Time.utc(2026, 6, 3, 20, 1, 30))
|
||||
expect(t).to eq(Time.utc(2026, 6, 3, 20, 0, 20))
|
||||
end
|
||||
end
|
||||
|
||||
@@ -15,7 +15,7 @@ RSpec.describe Youtube::BroadcastService do
|
||||
travel_to Time.zone.parse("2026-06-03 20:00:00 UTC") do
|
||||
past = Time.zone.parse("2026-06-03 10:00:00")
|
||||
t = service.send(:normalize_scheduled_start_time, past)
|
||||
expect(t).to eq(Time.utc(2026, 6, 3, 20, 1, 30))
|
||||
expect(t).to eq(Time.utc(2026, 6, 3, 20, 0, 20))
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user