module Ops class IncidentRecorder COOLDOWN = -> { ENV.fetch("OPS_NOTIFY_COOLDOWN_MINUTES", "30").to_i.minutes } class << self def record(finding) new.record(finding) end def resolve(fingerprint:) new.resolve(fingerprint: fingerprint) end end def record(finding) fingerprint = finding[:fingerprint] || fingerprint_for(finding) incident = Ops::Incident.open.find_by(fingerprint: fingerprint) if incident incident.occurrence_count += 1 incident.last_seen_at = Time.current incident.title = finding[:title] if finding[:title].present? incident.message = finding[:message] if finding[:message].present? incident.metadata = (incident.metadata || {}).merge(finding[:metadata] || {}) incident.severity = finding[:severity] if finding[:severity].present? incident.status = "open" if incident.status == "acknowledged" else now = Time.current incident = Ops::Incident.new( kind: finding[:kind], severity: finding[:severity], status: "open", title: finding[:title], message: finding[:message], metadata: finding[:metadata] || {}, fingerprint: fingerprint, occurrence_count: 1, first_seen_at: now, last_seen_at: now ) end incident.save! maybe_notify!(incident) incident end def resolve(fingerprint:) Ops::Incident.open.where(fingerprint: fingerprint).find_each(&:resolve!) end private def fingerprint_for(finding) Digest::SHA256.hexdigest([finding[:kind], finding[:title]].join("|")) end def maybe_notify!(incident) return if incident.muted? return unless Ops::Notifier.notify_severity?(incident.severity) return if incident.notified_at.present? && incident.notified_at > COOLDOWN.call.ago Ops::Notifier.new.notify(incident) incident.update_column(:notified_at, Time.current) end end end