Aggiunge monitoraggio ops: health check, log scanner, dashboard e notifiche.
Introduce incidenti con dedup, job Sidekiq ogni 3 min, scan log cron, /up/deep, dashboard /admin/ops e push ntfy; include Sentry opzionale e fix test suite. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
69
backend/spec/services/ops/health_checks_spec.rb
Normal file
69
backend/spec/services/ops/health_checks_spec.rb
Normal file
@@ -0,0 +1,69 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::HealthChecks do
|
||||
describe "#call" do
|
||||
it "risolve incidenti quando il disco è sano" do
|
||||
Ops::Incident.create!(
|
||||
kind: "disk_space",
|
||||
severity: "warning",
|
||||
status: "open",
|
||||
title: "Disco",
|
||||
message: "alto",
|
||||
metadata: {},
|
||||
fingerprint: "disk_space:root",
|
||||
occurrence_count: 1,
|
||||
first_seen_at: Time.current,
|
||||
last_seen_at: Time.current
|
||||
)
|
||||
|
||||
allow_any_instance_of(described_class).to receive(:check_disk_root).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "disk_space", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "disk_space:root"
|
||||
)
|
||||
)
|
||||
%i[
|
||||
check_recordings_size check_postgres check_redis check_mediamtx check_garage
|
||||
check_sidekiq_heartbeat check_sidekiq_dead check_http_public
|
||||
].each do |method|
|
||||
allow_any_instance_of(described_class).to receive(method).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "test", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "#{method}:ok"
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
described_class.new.call
|
||||
|
||||
expect(Ops::Incident.find_by(fingerprint: "disk_space:root").status).to eq("resolved")
|
||||
end
|
||||
end
|
||||
|
||||
describe "#summary" do
|
||||
it "restituisce status ok quando tutti i check sono sani" do
|
||||
allow_any_instance_of(described_class).to receive(:check_disk_root).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "disk_space", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "disk_space:root"
|
||||
)
|
||||
)
|
||||
%i[
|
||||
check_recordings_size check_postgres check_redis check_mediamtx check_garage
|
||||
check_sidekiq_heartbeat check_sidekiq_dead check_http_public
|
||||
].each do |method|
|
||||
allow_any_instance_of(described_class).to receive(method).and_return(
|
||||
described_class::Finding.new(
|
||||
kind: "test", severity: "info", healthy: true,
|
||||
title: "OK", message: "OK", metadata: {}, fingerprint: "#{method}:ok"
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
summary = described_class.new.summary
|
||||
|
||||
expect(summary[:status]).to eq("ok")
|
||||
expect(summary[:checks].size).to eq(9)
|
||||
end
|
||||
end
|
||||
end
|
||||
55
backend/spec/services/ops/incident_recorder_spec.rb
Normal file
55
backend/spec/services/ops/incident_recorder_spec.rb
Normal file
@@ -0,0 +1,55 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::IncidentRecorder do
|
||||
describe ".record" do
|
||||
it "crea un incidente nuovo" do
|
||||
incident = described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "critical",
|
||||
title: "Disco pieno",
|
||||
message: "90% usato",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
|
||||
expect(incident).to be_persisted
|
||||
expect(incident.occurrence_count).to eq(1)
|
||||
expect(incident.status).to eq("open")
|
||||
end
|
||||
|
||||
it "deduplica per fingerprint incrementando occorrenze" do
|
||||
described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "critical",
|
||||
title: "Disco pieno",
|
||||
message: "90%",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
second = described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "critical",
|
||||
title: "Disco pieno",
|
||||
message: "91%",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
|
||||
expect(Ops::Incident.where(fingerprint: "disk_space:root").count).to eq(1)
|
||||
expect(second.occurrence_count).to eq(2)
|
||||
end
|
||||
end
|
||||
|
||||
describe ".resolve" do
|
||||
it "chiude incidenti aperti con la stessa fingerprint" do
|
||||
incident = described_class.record(
|
||||
kind: "disk_space",
|
||||
severity: "warning",
|
||||
title: "Test",
|
||||
message: "msg",
|
||||
fingerprint: "disk_space:root"
|
||||
)
|
||||
|
||||
described_class.resolve(fingerprint: "disk_space:root")
|
||||
|
||||
expect(incident.reload.status).to eq("resolved")
|
||||
end
|
||||
end
|
||||
end
|
||||
24
backend/spec/services/ops/log_pattern_matcher_spec.rb
Normal file
24
backend/spec/services/ops/log_pattern_matcher_spec.rb
Normal file
@@ -0,0 +1,24 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::LogPatternMatcher do
|
||||
subject(:matcher) do
|
||||
described_class.new(
|
||||
patterns: [
|
||||
{ name: "http_500", severity: "warning", regex: /Completed 500/i },
|
||||
{ name: "disk_full", severity: "critical", regex: /No space left on device/i }
|
||||
]
|
||||
)
|
||||
end
|
||||
|
||||
it "trova pattern nei log" do
|
||||
log = <<~LOG
|
||||
I, [2026-06-10T19:45:10] Completed 500 Internal Server Error
|
||||
E, [2026-06-10T19:45:11] No space left on device - /recordings
|
||||
LOG
|
||||
|
||||
matches = matcher.scan(log)
|
||||
|
||||
expect(matches.map(&:name)).to contain_exactly("http_500", "disk_full")
|
||||
expect(matches.find { |m| m.name == "disk_full" }.severity).to eq("critical")
|
||||
end
|
||||
end
|
||||
9
backend/spec/services/ops/log_scanner_spec.rb
Normal file
9
backend/spec/services/ops/log_scanner_spec.rb
Normal file
@@ -0,0 +1,9 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::LogScanner do
|
||||
it "crea incidenti dai pattern trovati" do
|
||||
log = Rails.root.join("spec/fixtures/ops/sample.log").read
|
||||
|
||||
expect { described_class.new.ingest(log) }.to change(Ops::Incident, :count).by_at_least(2)
|
||||
end
|
||||
end
|
||||
43
backend/spec/services/ops/notifier_spec.rb
Normal file
43
backend/spec/services/ops/notifier_spec.rb
Normal file
@@ -0,0 +1,43 @@
|
||||
require "rails_helper"
|
||||
|
||||
RSpec.describe Ops::Notifier do
|
||||
let(:incident) do
|
||||
Ops::Incident.create!(
|
||||
kind: "test",
|
||||
severity: "critical",
|
||||
status: "open",
|
||||
title: "Test alert",
|
||||
message: "Messaggio di prova",
|
||||
metadata: {},
|
||||
fingerprint: "test:#{SecureRandom.hex(4)}",
|
||||
occurrence_count: 1,
|
||||
first_seen_at: Time.current,
|
||||
last_seen_at: Time.current
|
||||
)
|
||||
end
|
||||
|
||||
describe "#notify" do
|
||||
it "invia POST a ntfy quando configurato" do
|
||||
conn = instance_double(Faraday::Connection)
|
||||
response = instance_double(Faraday::Response, success?: true, status: 200, body: "")
|
||||
allow(Faraday).to receive(:new).and_return(conn)
|
||||
allow(conn).to receive(:post).and_return(response)
|
||||
allow(ENV).to receive(:[]).and_call_original
|
||||
allow(ENV).to receive(:[]).with("OPS_NTFY_URL").and_return("https://ntfy.example/ops")
|
||||
|
||||
described_class.new.notify(incident)
|
||||
|
||||
expect(conn).to have_received(:post).with("https://ntfy.example/ops", anything, hash_including("Title"))
|
||||
end
|
||||
end
|
||||
|
||||
describe ".notify_severity?" do
|
||||
it "rispetta OPS_NOTIFY_SEVERITIES" do
|
||||
allow(ENV).to receive(:fetch).and_call_original
|
||||
allow(ENV).to receive(:fetch).with("OPS_NOTIFY_SEVERITIES", "critical").and_return("critical,warning")
|
||||
|
||||
expect(described_class.notify_severity?("critical")).to be(true)
|
||||
expect(described_class.notify_severity?("info")).to be(false)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user