Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
67
backend/app/services/ops/incident_recorder.rb
Normal file
67
backend/app/services/ops/incident_recorder.rb
Normal file
@@ -0,0 +1,67 @@
|
||||
module Ops
|
||||
class IncidentRecorder
|
||||
COOLDOWN = -> { ENV.fetch("OPS_NOTIFY_COOLDOWN_MINUTES", "30").to_i.minutes }
|
||||
|
||||
class << self
|
||||
def record(finding)
|
||||
new.record(finding)
|
||||
end
|
||||
|
||||
def resolve(fingerprint:)
|
||||
new.resolve(fingerprint: fingerprint)
|
||||
end
|
||||
end
|
||||
|
||||
def record(finding)
|
||||
fingerprint = finding[:fingerprint] || fingerprint_for(finding)
|
||||
incident = Ops::Incident.open.find_by(fingerprint: fingerprint)
|
||||
|
||||
if incident
|
||||
incident.occurrence_count += 1
|
||||
incident.last_seen_at = Time.current
|
||||
incident.title = finding[:title] if finding[:title].present?
|
||||
incident.message = finding[:message] if finding[:message].present?
|
||||
incident.metadata = (incident.metadata || {}).merge(finding[:metadata] || {})
|
||||
incident.severity = finding[:severity] if finding[:severity].present?
|
||||
incident.status = "open" if incident.status == "acknowledged"
|
||||
else
|
||||
now = Time.current
|
||||
incident = Ops::Incident.new(
|
||||
kind: finding[:kind],
|
||||
severity: finding[:severity],
|
||||
status: "open",
|
||||
title: finding[:title],
|
||||
message: finding[:message],
|
||||
metadata: finding[:metadata] || {},
|
||||
fingerprint: fingerprint,
|
||||
occurrence_count: 1,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now
|
||||
)
|
||||
end
|
||||
|
||||
incident.save!
|
||||
maybe_notify!(incident)
|
||||
incident
|
||||
end
|
||||
|
||||
def resolve(fingerprint:)
|
||||
Ops::Incident.open.where(fingerprint: fingerprint).find_each(&:resolve!)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def fingerprint_for(finding)
|
||||
Digest::SHA256.hexdigest([finding[:kind], finding[:title]].join("|"))
|
||||
end
|
||||
|
||||
def maybe_notify!(incident)
|
||||
return if incident.muted?
|
||||
return unless Ops::Notifier.notify_severity?(incident.severity)
|
||||
return if incident.notified_at.present? && incident.notified_at > COOLDOWN.call.ago
|
||||
|
||||
Ops::Notifier.new.notify(incident)
|
||||
incident.update_column(:notified_at, Time.current)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user