Completa il hardening autoscaler (Fase 4): budget, kill-switch e runbook.

Drain sicuro in scale-in, alert Ops su overflow e controlli admin senza abilitare il deploy in produzione.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-09 20:00:15 +02:00
co-authored by Cursor
parent 318a319608
commit e5fc925bae
15 changed files with 286 additions and 14 deletions
+54 -1
View File
@@ -44,7 +44,8 @@ module Ops
check_sidekiq_heartbeat,
check_sidekiq_dead,
check_http_rails,
check_rails_latency
check_rails_latency,
check_stream_overflow
]
findings << check_http_public if public_check_due?
findings
@@ -161,6 +162,58 @@ module Ops
fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
end
def check_stream_overflow
return ok_finding("stream_overflow:skip", "Nodi stream non migrati") unless ActiveRecord::Base.connection.data_source_exists?("stream_nodes")
orphan_hours = ENV.fetch("STREAM_OVERFLOW_ORPHAN_HOURS", "3").to_i
orphans = StreamNode.where.not(slug: "home").where(status: %w[ready draining]).select do |n|
n.active_publishers.zero? && n.created_at < orphan_hours.hours.ago
end
metrics = Streams::Autoscaler.metrics
over_budget = !metrics[:within_budget]
at_max = metrics[:overflow_nodes] >= metrics[:max_overflow_nodes] && metrics[:free_slots] <= metrics[:soft_free_slots]
if orphans.any?
return Finding.new(
kind: "stream_overflow",
severity: "warning",
healthy: false,
title: "Nodi stream overflow idle",
message: "#{orphans.size} nodo/i idle da >#{orphan_hours}h: #{orphans.map(&:slug).join(', ')}",
metadata: { "slugs" => orphans.map(&:slug) },
fingerprint: "stream_overflow:orphan_idle"
)
end
if over_budget
return Finding.new(
kind: "stream_overflow",
severity: "warning",
healthy: false,
title: "Budget overflow streaming",
message: "Stima €#{metrics[:estimated_monthly_eur]}/mese > budget €#{metrics[:monthly_budget_eur]}",
metadata: metrics.transform_keys(&:to_s),
fingerprint: "stream_overflow:budget"
)
end
if at_max
return Finding.new(
kind: "stream_overflow",
severity: "warning",
healthy: false,
title: "Capacità stream al massimo",
message: "overflow=#{metrics[:overflow_nodes]}/#{metrics[:max_overflow_nodes]} free_slots=#{metrics[:free_slots]}",
metadata: metrics.transform_keys(&:to_s),
fingerprint: "stream_overflow:at_max"
)
end
ok_finding("stream_overflow:ok", "Overflow streaming OK")
rescue StandardError => e
fail_finding("stream_overflow", "warning", "stream_overflow:error", "Check overflow fallito", e.message)
end
def check_sidekiq_heartbeat
redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i