Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -13,7 +13,22 @@ module Admin
|
||||
def metrics
|
||||
host = HostMetrics.new.sample!
|
||||
stats = DashboardStats.new.call
|
||||
render json: { host: host, stats: stats, at: Time.current.iso8601 }
|
||||
render json: {
|
||||
host: host,
|
||||
stats: stats,
|
||||
ops_summary: ops_summary,
|
||||
at: Time.current.iso8601
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ops_summary
|
||||
{
|
||||
open_critical: Ops::Incident.critical_open.count,
|
||||
open_warning: Ops::Incident.open.where(severity: "warning").count,
|
||||
open_total: Ops::Incident.open.count
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
37
backend/app/controllers/admin/ops_controller.rb
Normal file
37
backend/app/controllers/admin/ops_controller.rb
Normal file
@@ -0,0 +1,37 @@
|
||||
module Admin
|
||||
class OpsController < Admin::BaseController
|
||||
def index
|
||||
@open_incidents = Ops::Incident.open.recent.limit(50)
|
||||
@resolved_incidents = Ops::Incident.where(status: "resolved").since(30.days.ago).recent.limit(30)
|
||||
@summary = ops_summary
|
||||
end
|
||||
|
||||
def acknowledge
|
||||
incident = Ops::Incident.find(params[:id])
|
||||
incident.acknowledge!
|
||||
redirect_to admin_ops_path, notice: "Incidente preso in carico"
|
||||
end
|
||||
|
||||
def resolve
|
||||
incident = Ops::Incident.find(params[:id])
|
||||
incident.resolve!
|
||||
redirect_to admin_ops_path, notice: "Incidente risolto"
|
||||
end
|
||||
|
||||
def mute
|
||||
incident = Ops::Incident.find(params[:id])
|
||||
incident.mute!(duration: 24.hours)
|
||||
redirect_to admin_ops_path, notice: "Notifiche sospese per 24 ore"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def ops_summary
|
||||
{
|
||||
open_critical: Ops::Incident.critical_open.count,
|
||||
open_warning: Ops::Incident.open.where(severity: "warning").count,
|
||||
open_total: Ops::Incident.open.count
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
28
backend/app/controllers/ops/health_controller.rb
Normal file
28
backend/app/controllers/ops/health_controller.rb
Normal file
@@ -0,0 +1,28 @@
|
||||
module Ops
|
||||
class HealthController < ActionController::API
|
||||
before_action :authorize_deep_health!
|
||||
|
||||
def show
|
||||
summary = Ops::HealthChecks.new.summary
|
||||
status_code = case summary[:status]
|
||||
when "ok" then :ok
|
||||
when "degraded" then :service_unavailable
|
||||
else :service_unavailable
|
||||
end
|
||||
|
||||
render json: summary, status: status_code
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def authorize_deep_health!
|
||||
token = ENV["OPS_HEALTH_TOKEN"].presence
|
||||
return if token.blank?
|
||||
|
||||
provided = request.headers["X-Ops-Token"].presence || params[:token].presence
|
||||
return head :unauthorized if provided.blank?
|
||||
|
||||
head :unauthorized unless ActiveSupport::SecurityUtils.secure_compare(provided, token)
|
||||
end
|
||||
end
|
||||
end
|
||||
33
backend/app/jobs/ops/health_monitor_job.rb
Normal file
33
backend/app/jobs/ops/health_monitor_job.rb
Normal file
@@ -0,0 +1,33 @@
|
||||
module Ops
|
||||
class HealthMonitorJob
|
||||
include Sidekiq::Job
|
||||
|
||||
sidekiq_options retry: 1, queue: "default"
|
||||
|
||||
INTERVAL_SECS = ENV.fetch("OPS_HEALTH_INTERVAL_SECS", "180").to_i
|
||||
REDIS_CHAIN_KEY = "ops:health_monitor:chain"
|
||||
HEARTBEAT_KEY = "ops:health_monitor:last_run"
|
||||
|
||||
def self.ensure_chain
|
||||
return unless redis
|
||||
|
||||
return if redis.get(REDIS_CHAIN_KEY)
|
||||
|
||||
redis.set(REDIS_CHAIN_KEY, "1", ex: INTERVAL_SECS * 2)
|
||||
perform_in(INTERVAL_SECS)
|
||||
end
|
||||
|
||||
def self.redis
|
||||
@redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
rescue Redis::CannotConnectError
|
||||
nil
|
||||
end
|
||||
|
||||
def perform
|
||||
self.class.redis&.set(HEARTBEAT_KEY, Time.current.to_i, ex: INTERVAL_SECS * 3)
|
||||
Ops::HealthChecks.new.call
|
||||
ensure
|
||||
self.class.perform_in(INTERVAL_SECS)
|
||||
end
|
||||
end
|
||||
end
|
||||
11
backend/app/mailers/ops/alert_mailer.rb
Normal file
11
backend/app/mailers/ops/alert_mailer.rb
Normal file
@@ -0,0 +1,11 @@
|
||||
module Ops
|
||||
class AlertMailer < ApplicationMailer
|
||||
def incident(incident)
|
||||
@incident = incident
|
||||
mail(
|
||||
to: ENV.fetch("OPS_ALERT_EMAIL"),
|
||||
subject: "[Match Live TV Ops] #{incident.severity.upcase}: #{incident.title}"
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
43
backend/app/models/ops/incident.rb
Normal file
43
backend/app/models/ops/incident.rb
Normal file
@@ -0,0 +1,43 @@
|
||||
module Ops
|
||||
class Incident < ApplicationRecord
|
||||
self.table_name = "ops_incidents"
|
||||
|
||||
KINDS = %w[
|
||||
disk_space recordings_size service_down http_public sidekiq_stale sidekiq_dead
|
||||
log_pattern garage_storage
|
||||
].freeze
|
||||
SEVERITIES = %w[critical warning info].freeze
|
||||
STATUSES = %w[open acknowledged resolved].freeze
|
||||
|
||||
validates :kind, :severity, :status, :title, :fingerprint, presence: true
|
||||
validates :severity, inclusion: { in: SEVERITIES }
|
||||
validates :status, inclusion: { in: STATUSES }
|
||||
validates :occurrence_count, numericality: { greater_than: 0 }
|
||||
|
||||
scope :open, -> { where(status: %w[open acknowledged]) }
|
||||
scope :unresolved, -> { where.not(status: "resolved") }
|
||||
scope :critical_open, -> { open.where(severity: "critical") }
|
||||
scope :recent, -> { order(last_seen_at: :desc) }
|
||||
scope :since, ->(time) { where("last_seen_at >= ?", time) }
|
||||
|
||||
def open?
|
||||
status.in?(%w[open acknowledged])
|
||||
end
|
||||
|
||||
def muted?
|
||||
muted_until.present? && muted_until.future?
|
||||
end
|
||||
|
||||
def acknowledge!
|
||||
update!(status: "acknowledged", acknowledged_at: Time.current)
|
||||
end
|
||||
|
||||
def resolve!
|
||||
update!(status: "resolved")
|
||||
end
|
||||
|
||||
def mute!(duration: 24.hours)
|
||||
update!(muted_until: duration.from_now)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -84,26 +84,57 @@ module Admin
|
||||
def disk_usage_for(path)
|
||||
return unavailable_disk unless path.present? && (Dir.exist?(path) || File.exist?(path))
|
||||
|
||||
usage = disk_usage_via_statvfs(path) || disk_usage_via_df(path)
|
||||
return unavailable_disk unless usage
|
||||
|
||||
usage.merge(dir_size_bytes: recordings_dir_size(path))
|
||||
rescue StandardError
|
||||
unavailable_disk
|
||||
end
|
||||
|
||||
def disk_usage_via_statvfs(path)
|
||||
return nil unless File.respond_to?(:statvfs)
|
||||
|
||||
stat = File.statvfs(path)
|
||||
total = stat.blocks * stat.frsize
|
||||
free = stat.bavail * stat.frsize
|
||||
used = total - free
|
||||
build_disk_usage(path, total: total, used: used, free: free)
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
|
||||
def disk_usage_via_df(path)
|
||||
output = shell_out("df -B1 --output=size,used,avail,pcent,target #{Shellwords.escape(path)} 2>/dev/null").strip
|
||||
line = output.lines.map(&:strip).reject(&:blank?).last
|
||||
return nil if line.blank? || line.start_with?("Filesystem")
|
||||
|
||||
parts = line.split
|
||||
return nil if parts.size < 5
|
||||
|
||||
total = parts[0].to_i
|
||||
used = parts[1].to_i
|
||||
free = parts[2].to_i
|
||||
pct = parts[3].delete("%").to_f
|
||||
build_disk_usage(path, total: total, used: used, free: free, used_percent: pct)
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
|
||||
def build_disk_usage(path, total:, used:, free:, used_percent: nil)
|
||||
{
|
||||
path: path,
|
||||
total_bytes: total,
|
||||
used_bytes: used,
|
||||
free_bytes: free,
|
||||
used_percent: total.positive? ? ((used.to_f / total) * 100).round(1) : 0.0,
|
||||
dir_size_bytes: recordings_dir_size(path)
|
||||
used_percent: used_percent || (total.positive? ? ((used.to_f / total) * 100).round(1) : 0.0)
|
||||
}
|
||||
rescue StandardError
|
||||
unavailable_disk
|
||||
end
|
||||
|
||||
def recordings_dir_size(path)
|
||||
return nil unless path == RECORDINGS_PATH && Dir.exist?(path)
|
||||
|
||||
`du -sb #{Shellwords.escape(path)} 2>/dev/null`.split.first.to_i
|
||||
shell_out("du -sb #{Shellwords.escape(path)} 2>/dev/null").split.first.to_i
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
@@ -112,6 +143,10 @@ module Admin
|
||||
{ path: nil, total_bytes: nil, used_bytes: nil, free_bytes: nil, used_percent: nil, dir_size_bytes: nil }
|
||||
end
|
||||
|
||||
def shell_out(cmd)
|
||||
`#{cmd}`
|
||||
end
|
||||
|
||||
def read_network
|
||||
rx, tx = parse_net_bytes
|
||||
prev = MetricsStore.net_totals
|
||||
|
||||
248
backend/app/services/ops/health_checks.rb
Normal file
248
backend/app/services/ops/health_checks.rb
Normal file
@@ -0,0 +1,248 @@
|
||||
module Ops
|
||||
class HealthChecks
|
||||
Finding = Struct.new(:kind, :severity, :healthy, :title, :message, :metadata, :fingerprint, keyword_init: true)
|
||||
|
||||
DISK_WARN_PERCENT = -> { ENV.fetch("OPS_DISK_WARN_PERCENT", "80").to_f }
|
||||
DISK_CRIT_PERCENT = -> { ENV.fetch("OPS_DISK_CRIT_PERCENT", "90").to_f }
|
||||
RECORDINGS_WARN_BYTES = -> { ENV.fetch("OPS_RECORDINGS_WARN_GB", "20").to_i.gigabytes }
|
||||
RECORDINGS_CRIT_BYTES = -> { ENV.fetch("OPS_RECORDINGS_CRIT_GB", "50").to_i.gigabytes }
|
||||
SIDEKIQ_STALE_SECS = -> { ENV.fetch("OPS_SIDEKIQ_STALE_SECS", "300").to_i }
|
||||
|
||||
def call
|
||||
findings = [
|
||||
check_disk_root,
|
||||
check_recordings_size,
|
||||
check_postgres,
|
||||
check_redis,
|
||||
check_mediamtx,
|
||||
check_garage,
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_public
|
||||
]
|
||||
|
||||
findings.each do |finding|
|
||||
if finding.healthy
|
||||
Ops::IncidentRecorder.resolve(fingerprint: finding.fingerprint)
|
||||
else
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: finding.kind,
|
||||
severity: finding.severity,
|
||||
title: finding.title,
|
||||
message: finding.message,
|
||||
metadata: finding.metadata,
|
||||
fingerprint: finding.fingerprint
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
findings
|
||||
end
|
||||
|
||||
def summary
|
||||
findings = [
|
||||
check_disk_root,
|
||||
check_recordings_size,
|
||||
check_postgres,
|
||||
check_redis,
|
||||
check_mediamtx,
|
||||
check_garage,
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_public
|
||||
]
|
||||
|
||||
status = if findings.any? { |f| !f.healthy && f.severity == "critical" }
|
||||
"down"
|
||||
elsif findings.any? { |f| !f.healthy }
|
||||
"degraded"
|
||||
else
|
||||
"ok"
|
||||
end
|
||||
|
||||
{
|
||||
status: status,
|
||||
checked_at: Time.current.iso8601,
|
||||
checks: findings.map { |f| finding_to_hash(f) }
|
||||
}
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def check_disk_root
|
||||
disk = Admin::HostMetrics.new.send(:read_disk)[:root]
|
||||
pct = disk[:used_percent].to_f
|
||||
healthy = pct < DISK_WARN_PERCENT.call
|
||||
severity = pct >= DISK_CRIT_PERCENT.call ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "disk_space",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Disco sistema OK" : "Disco sistema quasi pieno",
|
||||
message: "Uso disco #{pct}% (#{format_bytes(disk[:used_bytes])} / #{format_bytes(disk[:total_bytes])})",
|
||||
metadata: disk.transform_keys(&:to_s),
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
end
|
||||
|
||||
def check_recordings_size
|
||||
path = MatchLiveTv.recordings_local_path
|
||||
disk = Admin::HostMetrics.new.send(:disk_usage_for, path)
|
||||
size = disk[:dir_size_bytes].to_i
|
||||
healthy = size < RECORDINGS_WARN_BYTES.call
|
||||
severity = size >= RECORDINGS_CRIT_BYTES.call ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "recordings_size",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Archivio registrazioni OK" : "Archivio registrazioni elevato",
|
||||
message: "Contenuto #{path}: #{format_bytes(size)}",
|
||||
metadata: { "path" => path, "dir_size_bytes" => size },
|
||||
fingerprint: "recordings_size:local"
|
||||
)
|
||||
end
|
||||
|
||||
def check_postgres
|
||||
ActiveRecord::Base.connection.execute("SELECT 1")
|
||||
ok_finding("service_down:postgres", "PostgreSQL OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:postgres", "PostgreSQL non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_redis
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
redis.ping
|
||||
ok_finding("service_down:redis", "Redis OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:redis", "Redis non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_mediamtx
|
||||
Mediamtx::Client.new.list_paths
|
||||
ok_finding("service_down:mediamtx", "MediaMTX OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("service_down", "critical", "service_down:mediamtx", "MediaMTX API non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_garage
|
||||
return ok_finding("garage_storage:skip", "Garage non configurato") if MatchLiveTv.replay_storage_local?
|
||||
|
||||
require "aws-sdk-s3"
|
||||
client = Aws::S3::Client.new(
|
||||
endpoint: MatchLiveTv.replay_storage_endpoint,
|
||||
region: MatchLiveTv.replay_storage_region,
|
||||
access_key_id: MatchLiveTv.replay_storage_access_key_id,
|
||||
secret_access_key: MatchLiveTv.replay_storage_secret_access_key,
|
||||
force_path_style: MatchLiveTv.replay_storage_force_path_style?
|
||||
)
|
||||
client.head_bucket(bucket: MatchLiveTv.replay_storage_bucket)
|
||||
ok_finding("garage_storage:ok", "Garage storage OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_sidekiq_heartbeat
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i
|
||||
age = last.positive? ? Time.current.to_i - last : nil
|
||||
healthy = age.present? && age < SIDEKIQ_STALE_SECS.call
|
||||
|
||||
Finding.new(
|
||||
kind: "sidekiq_stale",
|
||||
severity: "critical",
|
||||
healthy: healthy,
|
||||
title: healthy ? "Sidekiq attivo" : "Sidekiq non risponde",
|
||||
message: healthy ? "Ultimo health check #{age}s fa" : "Heartbeat assente o scaduto (#{age || 'mai'}s)",
|
||||
metadata: { "last_heartbeat_age_secs" => age },
|
||||
fingerprint: "sidekiq_stale:heartbeat"
|
||||
)
|
||||
end
|
||||
|
||||
def check_sidekiq_dead
|
||||
dead_count = Sidekiq::DeadSet.new.size
|
||||
healthy = dead_count.zero?
|
||||
severity = dead_count > 5 ? "critical" : "warning"
|
||||
|
||||
Finding.new(
|
||||
kind: "sidekiq_dead",
|
||||
severity: severity,
|
||||
healthy: healthy,
|
||||
title: healthy ? "Nessun job Sidekiq morto" : "Job Sidekiq in dead queue",
|
||||
message: "#{dead_count} job nella dead queue",
|
||||
metadata: { "dead_count" => dead_count },
|
||||
fingerprint: "sidekiq_dead:count"
|
||||
)
|
||||
end
|
||||
|
||||
def check_http_public
|
||||
return ok_finding("http_public:skip", "HTTP public check disabilitato in test") if Rails.env.test?
|
||||
|
||||
url = "#{MatchLiveTv.app_public_url.chomp('/')}/up"
|
||||
response = Faraday.get(url) { |req| req.options.timeout = 10 }
|
||||
healthy = response.status == 200
|
||||
|
||||
Finding.new(
|
||||
kind: "http_public",
|
||||
severity: "critical",
|
||||
healthy: healthy,
|
||||
title: healthy ? "Sito pubblico raggiungibile" : "Sito pubblico non risponde",
|
||||
message: healthy ? "GET #{url} → 200" : "GET #{url} → #{response.status}",
|
||||
metadata: { "url" => url, "status" => response.status },
|
||||
fingerprint: "http_public:up"
|
||||
)
|
||||
rescue Faraday::Error => e
|
||||
fail_finding("http_public", "critical", "http_public:up", "Sito pubblico non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def ok_finding(fingerprint, title)
|
||||
Finding.new(
|
||||
kind: fingerprint.split(":").first,
|
||||
severity: "info",
|
||||
healthy: true,
|
||||
title: title,
|
||||
message: title,
|
||||
metadata: {},
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
end
|
||||
|
||||
def fail_finding(kind, severity, fingerprint, title, message)
|
||||
Finding.new(
|
||||
kind: kind,
|
||||
severity: severity,
|
||||
healthy: false,
|
||||
title: title,
|
||||
message: message,
|
||||
metadata: {},
|
||||
fingerprint: fingerprint
|
||||
)
|
||||
end
|
||||
|
||||
def finding_to_hash(finding)
|
||||
{
|
||||
kind: finding.kind,
|
||||
severity: finding.severity,
|
||||
healthy: finding.healthy,
|
||||
title: finding.title,
|
||||
message: finding.message,
|
||||
metadata: finding.metadata,
|
||||
fingerprint: finding.fingerprint
|
||||
}
|
||||
end
|
||||
|
||||
def format_bytes(bytes)
|
||||
return "—" if bytes.nil?
|
||||
|
||||
units = %w[B KB MB GB TB]
|
||||
size = bytes.to_f
|
||||
unit = units.shift
|
||||
while size >= 1024 && units.any?
|
||||
size /= 1024
|
||||
unit = units.shift
|
||||
end
|
||||
"#{size.round(size >= 10 ? 0 : 1)} #{unit}"
|
||||
end
|
||||
end
|
||||
end
|
||||
67
backend/app/services/ops/incident_recorder.rb
Normal file
67
backend/app/services/ops/incident_recorder.rb
Normal file
@@ -0,0 +1,67 @@
|
||||
module Ops
|
||||
class IncidentRecorder
|
||||
COOLDOWN = -> { ENV.fetch("OPS_NOTIFY_COOLDOWN_MINUTES", "30").to_i.minutes }
|
||||
|
||||
class << self
|
||||
def record(finding)
|
||||
new.record(finding)
|
||||
end
|
||||
|
||||
def resolve(fingerprint:)
|
||||
new.resolve(fingerprint: fingerprint)
|
||||
end
|
||||
end
|
||||
|
||||
def record(finding)
|
||||
fingerprint = finding[:fingerprint] || fingerprint_for(finding)
|
||||
incident = Ops::Incident.open.find_by(fingerprint: fingerprint)
|
||||
|
||||
if incident
|
||||
incident.occurrence_count += 1
|
||||
incident.last_seen_at = Time.current
|
||||
incident.title = finding[:title] if finding[:title].present?
|
||||
incident.message = finding[:message] if finding[:message].present?
|
||||
incident.metadata = (incident.metadata || {}).merge(finding[:metadata] || {})
|
||||
incident.severity = finding[:severity] if finding[:severity].present?
|
||||
incident.status = "open" if incident.status == "acknowledged"
|
||||
else
|
||||
now = Time.current
|
||||
incident = Ops::Incident.new(
|
||||
kind: finding[:kind],
|
||||
severity: finding[:severity],
|
||||
status: "open",
|
||||
title: finding[:title],
|
||||
message: finding[:message],
|
||||
metadata: finding[:metadata] || {},
|
||||
fingerprint: fingerprint,
|
||||
occurrence_count: 1,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now
|
||||
)
|
||||
end
|
||||
|
||||
incident.save!
|
||||
maybe_notify!(incident)
|
||||
incident
|
||||
end
|
||||
|
||||
def resolve(fingerprint:)
|
||||
Ops::Incident.open.where(fingerprint: fingerprint).find_each(&:resolve!)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fingerprint_for(finding)
|
||||
Digest::SHA256.hexdigest([finding[:kind], finding[:title]].join("|"))
|
||||
end
|
||||
|
||||
def maybe_notify!(incident)
|
||||
return if incident.muted?
|
||||
return unless Ops::Notifier.notify_severity?(incident.severity)
|
||||
return if incident.notified_at.present? && incident.notified_at > COOLDOWN.call.ago
|
||||
|
||||
Ops::Notifier.new.notify(incident)
|
||||
incident.update_column(:notified_at, Time.current)
|
||||
end
|
||||
end
|
||||
end
|
||||
37
backend/app/services/ops/log_pattern_matcher.rb
Normal file
37
backend/app/services/ops/log_pattern_matcher.rb
Normal file
@@ -0,0 +1,37 @@
|
||||
module Ops
|
||||
class LogPatternMatcher
|
||||
Match = Struct.new(:name, :severity, :line, keyword_init: true)
|
||||
|
||||
def initialize(patterns: nil)
|
||||
@patterns = patterns || load_patterns
|
||||
end
|
||||
|
||||
def scan(text)
|
||||
return [] if text.blank?
|
||||
|
||||
matches = []
|
||||
text.each_line do |line|
|
||||
@patterns.each do |pattern|
|
||||
next unless line.match?(pattern[:regex])
|
||||
|
||||
matches << Match.new(name: pattern[:name], severity: pattern[:severity], line: line.strip)
|
||||
end
|
||||
end
|
||||
matches
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def load_patterns
|
||||
path = Rails.root.join("config/ops_log_patterns.yml")
|
||||
raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: true) || {}
|
||||
Array(raw["patterns"]).map do |entry|
|
||||
{
|
||||
name: entry["name"].to_s,
|
||||
severity: entry["severity"].to_s,
|
||||
regex: Regexp.new(entry["regex"], Regexp::IGNORECASE)
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
26
backend/app/services/ops/log_scanner.rb
Normal file
26
backend/app/services/ops/log_scanner.rb
Normal file
@@ -0,0 +1,26 @@
|
||||
module Ops
|
||||
class LogScanner
|
||||
def initialize(matcher: LogPatternMatcher.new)
|
||||
@matcher = matcher
|
||||
end
|
||||
|
||||
def ingest(text)
|
||||
matches = @matcher.scan(text)
|
||||
matches.each { |match| record_match(match) }
|
||||
matches.size
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def record_match(match)
|
||||
Ops::IncidentRecorder.record(
|
||||
kind: "log_pattern",
|
||||
severity: match.severity,
|
||||
title: "Log: #{match.name}",
|
||||
message: match.line.truncate(500),
|
||||
metadata: { "pattern" => match.name, "sample" => match.line },
|
||||
fingerprint: "log_pattern:#{match.name}"
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
65
backend/app/services/ops/notifier.rb
Normal file
65
backend/app/services/ops/notifier.rb
Normal file
@@ -0,0 +1,65 @@
|
||||
module Ops
|
||||
class Notifier
|
||||
class Error < StandardError; end
|
||||
|
||||
def self.notify_severity?(severity)
|
||||
levels = ENV.fetch("OPS_NOTIFY_SEVERITIES", "critical").split(",").map(&:strip)
|
||||
levels.include?(severity.to_s)
|
||||
end
|
||||
|
||||
def notify(incident)
|
||||
notify_ntfy(incident) if ntfy_url.present?
|
||||
notify_email(incident) if alert_email.present?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def notify_ntfy(incident)
|
||||
priority = incident.severity == "critical" ? "5" : "3"
|
||||
tags = incident.severity == "critical" ? "rotating_light" : "warning"
|
||||
|
||||
conn = Faraday.new do |f|
|
||||
f.adapter Faraday.default_adapter
|
||||
f.options.timeout = 10
|
||||
f.options.open_timeout = 5
|
||||
end
|
||||
|
||||
headers = {
|
||||
"Title" => "[Match Live TV] #{incident.title}",
|
||||
"Priority" => priority,
|
||||
"Tags" => tags
|
||||
}
|
||||
headers["Authorization"] = "Bearer #{ntfy_token}" if ntfy_token.present?
|
||||
|
||||
body = incident.message.to_s
|
||||
body += "\n\n(#{incident.occurrence_count} occorrenze)" if incident.occurrence_count > 1
|
||||
|
||||
response = conn.post(ntfy_url, body, headers)
|
||||
return if response.success?
|
||||
|
||||
Rails.logger.warn("[Ops::Notifier] ntfy failed: #{response.status} #{response.body}")
|
||||
rescue Faraday::Error => e
|
||||
Rails.logger.warn("[Ops::Notifier] ntfy error: #{e.message}")
|
||||
end
|
||||
|
||||
def notify_email(incident)
|
||||
return unless defined?(ActionMailer)
|
||||
|
||||
Ops::AlertMailer.incident(incident).deliver_later
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn("[Ops::Notifier] email error: #{e.message}")
|
||||
end
|
||||
|
||||
def ntfy_url
|
||||
ENV["OPS_NTFY_URL"].presence
|
||||
end
|
||||
|
||||
def ntfy_token
|
||||
ENV["OPS_NTFY_TOKEN"].presence
|
||||
end
|
||||
|
||||
def alert_email
|
||||
ENV["OPS_ALERT_EMAIL"].presence
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -54,6 +54,19 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" style="margin-bottom:1.25rem">
|
||||
<h2>Stato piattaforma (Ops)</h2>
|
||||
<% critical = Ops::Incident.critical_open.count %>
|
||||
<% warnings = Ops::Incident.open.where(severity: "warning").count %>
|
||||
<% if critical.positive? %>
|
||||
<p class="admin-flash" style="margin-bottom:0.75rem"><%= critical %> incidente/i critico/i aperto/i — <%= link_to "Vedi dashboard Ops", admin_ops_path %></p>
|
||||
<% elsif warnings.positive? %>
|
||||
<p class="kpi-sub" style="margin-bottom:0.75rem"><%= warnings %> warning — <%= link_to "Dashboard Ops", admin_ops_path %></p>
|
||||
<% else %>
|
||||
<p class="empty" style="margin:0">Nessun incidente aperto. <%= link_to "Dashboard Ops", admin_ops_path %></p>
|
||||
<% end %>
|
||||
</section>
|
||||
|
||||
<section class="charts-grid" style="grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));">
|
||||
<div class="chart-card">
|
||||
<h3>Disco sistema</h3>
|
||||
|
||||
83
backend/app/views/admin/ops/index.html.erb
Normal file
83
backend/app/views/admin/ops/index.html.erb
Normal file
@@ -0,0 +1,83 @@
|
||||
<% content_for :body_class, "admin-body" %>
|
||||
|
||||
<section class="kpi-grid">
|
||||
<div class="kpi-card <%= @summary[:open_critical].positive? ? 'kpi-card--danger' : 'kpi-card--accent' %>">
|
||||
<div class="kpi-label">Critici aperti</div>
|
||||
<div class="kpi-value"><%= @summary[:open_critical] %></div>
|
||||
</div>
|
||||
<div class="kpi-card">
|
||||
<div class="kpi-label">Warning aperti</div>
|
||||
<div class="kpi-value"><%= @summary[:open_warning] %></div>
|
||||
</div>
|
||||
<div class="kpi-card">
|
||||
<div class="kpi-label">Totale aperti</div>
|
||||
<div class="kpi-value"><%= @summary[:open_total] %></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="panel" style="margin-bottom:1.5rem">
|
||||
<h2>Incidenti aperti</h2>
|
||||
<% if @open_incidents.any? %>
|
||||
<table class="admin-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Severità</th>
|
||||
<th>Tipo</th>
|
||||
<th>Titolo</th>
|
||||
<th>Occ.</th>
|
||||
<th>Ultimo</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<% @open_incidents.each do |inc| %>
|
||||
<tr>
|
||||
<td><span class="badge badge--<%= inc.severity == 'critical' ? 'live' : 'paused' %>"><%= inc.severity %></span></td>
|
||||
<td class="muted"><%= inc.kind %></td>
|
||||
<td>
|
||||
<strong><%= inc.title %></strong>
|
||||
<% if inc.message.present? %>
|
||||
<div class="muted" style="font-size:0.85rem;margin-top:0.25rem"><%= truncate(inc.message, length: 120) %></div>
|
||||
<% end %>
|
||||
</td>
|
||||
<td><%= inc.occurrence_count %></td>
|
||||
<td class="muted"><%= inc.last_seen_at&.strftime("%d/%m %H:%M") %></td>
|
||||
<td class="admin-actions">
|
||||
<% unless inc.status == 'acknowledged' %>
|
||||
<%= button_to "Preso in carico", acknowledge_admin_op_path(inc), method: :post, class: "admin-btn admin-btn--sm" %>
|
||||
<% end %>
|
||||
<%= button_to "Risolvi", resolve_admin_op_path(inc), method: :post, class: "admin-btn admin-btn--sm" %>
|
||||
<%= button_to "Mute 24h", mute_admin_op_path(inc), method: :post, class: "admin-btn admin-btn--sm admin-btn--outline" %>
|
||||
</td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</tbody>
|
||||
</table>
|
||||
<% else %>
|
||||
<p class="empty">Nessun incidente aperto. La piattaforma risulta sana.</p>
|
||||
<% end %>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<h2>Risolti (ultimi 30 giorni)</h2>
|
||||
<% if @resolved_incidents.any? %>
|
||||
<table class="admin-table">
|
||||
<thead>
|
||||
<tr><th>Severità</th><th>Tipo</th><th>Titolo</th><th>Occ.</th><th>Risolto</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<% @resolved_incidents.each do |inc| %>
|
||||
<tr>
|
||||
<td><span class="badge badge--connecting"><%= inc.severity %></span></td>
|
||||
<td class="muted"><%= inc.kind %></td>
|
||||
<td><%= inc.title %></td>
|
||||
<td><%= inc.occurrence_count %></td>
|
||||
<td class="muted"><%= inc.updated_at.strftime("%d/%m %H:%M") %></td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</tbody>
|
||||
</table>
|
||||
<% else %>
|
||||
<p class="empty">Nessun incidente risolto di recente.</p>
|
||||
<% end %>
|
||||
</div>
|
||||
@@ -16,6 +16,10 @@
|
||||
<nav class="admin-nav">
|
||||
<% if admin_logged_in? %>
|
||||
<%= link_to "Dashboard", admin_root_path, class: ("active" if controller_name == "dashboard") %>
|
||||
<% ops_critical = Ops::Incident.critical_open.count %>
|
||||
<%= link_to admin_ops_path, class: ("active" if controller_name == "ops") do %>
|
||||
Ops<% if ops_critical.positive? %> <span class="admin-nav-badge"><%= ops_critical %></span><% end %>
|
||||
<% end %>
|
||||
<%= link_to "Società e squadre", admin_clubs_path, class: ("active" if controller_name.in?(%w[clubs teams])) %>
|
||||
<%= link_to "Fatturazione", admin_billing_path, class: ("active" if controller_name.in?(%w[billing billing_invoices])) %>
|
||||
<%= link_to "YouTube", admin_youtube_platform_path, class: ("active" if controller_name == "youtube") %>
|
||||
|
||||
10
backend/app/views/ops/alert_mailer/incident.text.erb
Normal file
10
backend/app/views/ops/alert_mailer/incident.text.erb
Normal file
@@ -0,0 +1,10 @@
|
||||
<%= @incident.title %>
|
||||
|
||||
Severità: <%= @incident.severity %>
|
||||
Tipo: <%= @incident.kind %>
|
||||
Occorrenze: <%= @incident.occurrence_count %>
|
||||
Ultimo avvistamento: <%= @incident.last_seen_at %>
|
||||
|
||||
<%= @incident.message %>
|
||||
|
||||
Dashboard: <%= ENV.fetch("APP_PUBLIC_URL", "https://www.matchlivetv.it").chomp("/") %>/admin/ops
|
||||
Reference in New Issue
Block a user