Emiliano Frascaro
52cfffb83f
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
68 lines
2.0 KiB
Ruby
68 lines
2.0 KiB
Ruby
module Ops
|
|
class IncidentRecorder
|
|
COOLDOWN = -> { ENV.fetch("OPS_NOTIFY_COOLDOWN_MINUTES", "30").to_i.minutes }
|
|
|
|
class << self
|
|
def record(finding)
|
|
new.record(finding)
|
|
end
|
|
|
|
def resolve(fingerprint:)
|
|
new.resolve(fingerprint: fingerprint)
|
|
end
|
|
end
|
|
|
|
def record(finding)
|
|
fingerprint = finding[:fingerprint] || fingerprint_for(finding)
|
|
incident = Ops::Incident.open.find_by(fingerprint: fingerprint)
|
|
|
|
if incident
|
|
incident.occurrence_count += 1
|
|
incident.last_seen_at = Time.current
|
|
incident.title = finding[:title] if finding[:title].present?
|
|
incident.message = finding[:message] if finding[:message].present?
|
|
incident.metadata = (incident.metadata || {}).merge(finding[:metadata] || {})
|
|
incident.severity = finding[:severity] if finding[:severity].present?
|
|
incident.status = "open" if incident.status == "acknowledged"
|
|
else
|
|
now = Time.current
|
|
incident = Ops::Incident.new(
|
|
kind: finding[:kind],
|
|
severity: finding[:severity],
|
|
status: "open",
|
|
title: finding[:title],
|
|
message: finding[:message],
|
|
metadata: finding[:metadata] || {},
|
|
fingerprint: fingerprint,
|
|
occurrence_count: 1,
|
|
first_seen_at: now,
|
|
last_seen_at: now
|
|
)
|
|
end
|
|
|
|
incident.save!
|
|
maybe_notify!(incident)
|
|
incident
|
|
end
|
|
|
|
def resolve(fingerprint:)
|
|
Ops::Incident.open.where(fingerprint: fingerprint).find_each(&:resolve!)
|
|
end
|
|
|
|
private
|
|
|
|
def fingerprint_for(finding)
|
|
Digest::SHA256.hexdigest([finding[:kind], finding[:title]].join("|"))
|
|
end
|
|
|
|
def maybe_notify!(incident)
|
|
return if incident.muted?
|
|
return unless Ops::Notifier.notify_severity?(incident.severity)
|
|
return if incident.notified_at.present? && incident.notified_at > COOLDOWN.call.ago
|
|
|
|
Ops::Notifier.new.notify(incident)
|
|
incident.update_column(:notified_at, Time.current)
|
|
end
|
|
end
|
|
end
|