Completa il hardening autoscaler (Fase 4): budget, kill-switch e runbook.
Drain sicuro in scale-in, alert Ops su overflow e controlli admin senza abilitare il deploy in produzione. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -44,7 +44,8 @@ module Ops
|
||||
check_sidekiq_heartbeat,
|
||||
check_sidekiq_dead,
|
||||
check_http_rails,
|
||||
check_rails_latency
|
||||
check_rails_latency,
|
||||
check_stream_overflow
|
||||
]
|
||||
findings << check_http_public if public_check_due?
|
||||
findings
|
||||
@@ -161,6 +162,58 @@ module Ops
|
||||
fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
|
||||
end
|
||||
|
||||
def check_stream_overflow
|
||||
return ok_finding("stream_overflow:skip", "Nodi stream non migrati") unless ActiveRecord::Base.connection.data_source_exists?("stream_nodes")
|
||||
|
||||
orphan_hours = ENV.fetch("STREAM_OVERFLOW_ORPHAN_HOURS", "3").to_i
|
||||
orphans = StreamNode.where.not(slug: "home").where(status: %w[ready draining]).select do |n|
|
||||
n.active_publishers.zero? && n.created_at < orphan_hours.hours.ago
|
||||
end
|
||||
metrics = Streams::Autoscaler.metrics
|
||||
over_budget = !metrics[:within_budget]
|
||||
at_max = metrics[:overflow_nodes] >= metrics[:max_overflow_nodes] && metrics[:free_slots] <= metrics[:soft_free_slots]
|
||||
|
||||
if orphans.any?
|
||||
return Finding.new(
|
||||
kind: "stream_overflow",
|
||||
severity: "warning",
|
||||
healthy: false,
|
||||
title: "Nodi stream overflow idle",
|
||||
message: "#{orphans.size} nodo/i idle da >#{orphan_hours}h: #{orphans.map(&:slug).join(', ')}",
|
||||
metadata: { "slugs" => orphans.map(&:slug) },
|
||||
fingerprint: "stream_overflow:orphan_idle"
|
||||
)
|
||||
end
|
||||
|
||||
if over_budget
|
||||
return Finding.new(
|
||||
kind: "stream_overflow",
|
||||
severity: "warning",
|
||||
healthy: false,
|
||||
title: "Budget overflow streaming",
|
||||
message: "Stima €#{metrics[:estimated_monthly_eur]}/mese > budget €#{metrics[:monthly_budget_eur]}",
|
||||
metadata: metrics.transform_keys(&:to_s),
|
||||
fingerprint: "stream_overflow:budget"
|
||||
)
|
||||
end
|
||||
|
||||
if at_max
|
||||
return Finding.new(
|
||||
kind: "stream_overflow",
|
||||
severity: "warning",
|
||||
healthy: false,
|
||||
title: "Capacità stream al massimo",
|
||||
message: "overflow=#{metrics[:overflow_nodes]}/#{metrics[:max_overflow_nodes]} free_slots=#{metrics[:free_slots]}",
|
||||
metadata: metrics.transform_keys(&:to_s),
|
||||
fingerprint: "stream_overflow:at_max"
|
||||
)
|
||||
end
|
||||
|
||||
ok_finding("stream_overflow:ok", "Overflow streaming OK")
|
||||
rescue StandardError => e
|
||||
fail_finding("stream_overflow", "warning", "stream_overflow:error", "Check overflow fallito", e.message)
|
||||
end
|
||||
|
||||
def check_sidekiq_heartbeat
|
||||
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
|
||||
last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i
|
||||
|
||||
Reference in New Issue
Block a user