Completa il hardening autoscaler (Fase 4): budget, kill-switch e runbook.

Drain sicuro in scale-in, alert Ops su overflow e controlli admin senza abilitare il deploy in produzione.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/controllers/admin/stream_nodes_controller.rb

@@ -44,6 +44,16 @@ module Admin
       redirect_to admin_stream_nodes_path, alert: e.message
     end
 
+    def kill_switch
+      Streams::Autoscaler.engage_kill_switch!
+      redirect_to admin_stream_nodes_path, notice: t("admin.flash.autoscale_kill_on")
+    end
+
+    def clear_kill_switch
+      Streams::Autoscaler.clear_kill_switch!
+      redirect_to admin_stream_nodes_path, notice: t("admin.flash.autoscale_kill_off")
+    end
+
     private
 
     def lab_dns_snippet

backend/app/models/ops/incident.rb

@@ -4,7 +4,7 @@ module Ops
 
     KINDS = %w[
       disk_space recordings_size service_down http_public http_rails rails_latency
-      sidekiq_stale sidekiq_dead log_pattern garage_storage
+      sidekiq_stale sidekiq_dead log_pattern garage_storage stream_overflow
     ].freeze
     SEVERITIES = %w[critical warning info].freeze
     STATUSES = %w[open acknowledged resolved].freeze

backend/app/services/ops/health_checks.rb

@@ -44,7 +44,8 @@ module Ops
         check_sidekiq_heartbeat,
         check_sidekiq_dead,
         check_http_rails,
-        check_rails_latency
+        check_rails_latency,
+        check_stream_overflow
       ]
       findings << check_http_public if public_check_due?
       findings
@@ -161,6 +162,58 @@ module Ops
       fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
     end
 
+    def check_stream_overflow
+      return ok_finding("stream_overflow:skip", "Nodi stream non migrati") unless ActiveRecord::Base.connection.data_source_exists?("stream_nodes")
+
+      orphan_hours = ENV.fetch("STREAM_OVERFLOW_ORPHAN_HOURS", "3").to_i
+      orphans = StreamNode.where.not(slug: "home").where(status: %w[ready draining]).select do |n|
+        n.active_publishers.zero? && n.created_at < orphan_hours.hours.ago
+      end
+      metrics = Streams::Autoscaler.metrics
+      over_budget = !metrics[:within_budget]
+      at_max = metrics[:overflow_nodes] >= metrics[:max_overflow_nodes] && metrics[:free_slots] <= metrics[:soft_free_slots]
+
+      if orphans.any?
+        return Finding.new(
+          kind: "stream_overflow",
+          severity: "warning",
+          healthy: false,
+          title: "Nodi stream overflow idle",
+          message: "#{orphans.size} nodo/i idle da >#{orphan_hours}h: #{orphans.map(&:slug).join(', ')}",
+          metadata: { "slugs" => orphans.map(&:slug) },
+          fingerprint: "stream_overflow:orphan_idle"
+        )
+      end
+
+      if over_budget
+        return Finding.new(
+          kind: "stream_overflow",
+          severity: "warning",
+          healthy: false,
+          title: "Budget overflow streaming",
+          message: "Stima €#{metrics[:estimated_monthly_eur]}/mese > budget €#{metrics[:monthly_budget_eur]}",
+          metadata: metrics.transform_keys(&:to_s),
+          fingerprint: "stream_overflow:budget"
+        )
+      end
+
+      if at_max
+        return Finding.new(
+          kind: "stream_overflow",
+          severity: "warning",
+          healthy: false,
+          title: "Capacità stream al massimo",
+          message: "overflow=#{metrics[:overflow_nodes]}/#{metrics[:max_overflow_nodes]} free_slots=#{metrics[:free_slots]}",
+          metadata: metrics.transform_keys(&:to_s),
+          fingerprint: "stream_overflow:at_max"
+        )
+      end
+
+      ok_finding("stream_overflow:ok", "Overflow streaming OK")
+    rescue StandardError => e
+      fail_finding("stream_overflow", "warning", "stream_overflow:error", "Check overflow fallito", e.message)
+    end
+
     def check_sidekiq_heartbeat
       redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
       last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i

backend/app/services/streams/autoscaler.rb

@@ -2,15 +2,31 @@
 
 module Streams
   # Scale-out / warm spare / scale-in dei nodi overflow (lab o Hetzner).
-  # Kill-switch: STREAM_AUTOSCALE_ENABLED!=1 → no-op.
+  # Kill-switch: STREAM_AUTOSCALE_ENABLED!=1 OPPURE Redis streams:autoscaler:kill_switch=1.
   class Autoscaler
     Result = Struct.new(:actions, :metrics, :skipped, :error, keyword_init: true)
 
     LOCK_KEY = "streams:autoscaler:lock"
+    KILL_SWITCH_KEY = "streams:autoscaler:kill_switch"
 
     class << self
       def enabled?
-        ENV["STREAM_AUTOSCALE_ENABLED"] == "1"
+        return false if kill_switch_engaged?
+        return false unless ENV["STREAM_AUTOSCALE_ENABLED"] == "1"
+
+        true
+      end
+
+      def kill_switch_engaged?
+        redis_get(KILL_SWITCH_KEY) == "1"
+      end
+
+      def engage_kill_switch!
+        redis_set(KILL_SWITCH_KEY, "1")
+      end
+
+      def clear_kill_switch!
+        redis_del(KILL_SWITCH_KEY)
       end
 
       def soft_free_slots
@@ -33,6 +49,28 @@ module Streams
         ENV.fetch("STREAM_AUTOSCALE_KIND", "lab") # lab|cloud
       end
 
+      def allow_cloud?
+        ENV["STREAM_AUTOSCALE_ALLOW_CLOUD"] == "1" && ENV["HCLOUD_TOKEN"].present?
+      end
+
+      def node_eur_per_hour
+        ENV.fetch("STREAM_AUTOSCALE_NODE_EUR_PER_HOUR", "0.015").to_f
+      end
+
+      def monthly_budget_eur
+        ENV.fetch("STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR", "40").to_f
+      end
+
+      def estimated_monthly_eur(overflow_count = nil)
+        count = overflow_count || metrics[:overflow_nodes]
+        # Worst case: nodi sempre accesi 24/7
+        (count * node_eur_per_hour * 24 * 30).round(2)
+      end
+
+      def within_budget?(overflow_count = nil)
+        estimated_monthly_eur(overflow_count) <= monthly_budget_eur
+      end
+
       def reconcile!(provisioner: nil)
         return Result.new(skipped: true, actions: [], metrics: metrics) unless enabled?
 
@@ -62,13 +100,33 @@ module Streams
           warm_spare_min: warm_spare_min,
           max_overflow_nodes: max_overflow_nodes,
           enabled: enabled?,
-          kind: kind
+          env_enabled: ENV["STREAM_AUTOSCALE_ENABLED"] == "1",
+          kill_switch: kill_switch_engaged?,
+          kind: kind,
+          allow_cloud: allow_cloud?,
+          estimated_monthly_eur: estimated_monthly_eur(overflow.size),
+          monthly_budget_eur: monthly_budget_eur,
+          within_budget: within_budget?(overflow.size)
         }
       end
 
       def worker_id
         ENV.fetch("HOSTNAME", "autoscaler")
       end
+
+      def redis_get(key)
+        Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")).get(key)
+      rescue Redis::BaseError
+        nil
+      end
+
+      def redis_set(key, value)
+        Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")).set(key, value)
+      end
+
+      def redis_del(key)
+        Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")).del(key)
+      end
     end
 
     def initialize(provisioner: nil)
@@ -83,6 +141,9 @@ module Streams
         provision_overflow!
         actions << :scale_out
         m = self.class.metrics
+      elsif need_capacity?(m) && !can_provision?(m)
+        actions << :blocked_capacity
+        Rails.logger.warn("[Streams::Autoscaler] capacity needed but blocked metrics=#{m.inspect}")
       end
 
       if warm_spare_desired?(m) && m[:spare_ready] < self.class.warm_spare_min && can_provision?(m)
@@ -94,7 +155,7 @@ module Streams
       scale_in_candidates.each do |node|
         next if keep_as_warm_spare?(node)
 
-        @provisioner.decommission!(node)
+        safe_scale_in!(node)
         actions << :"scale_in_#{node.slug}"
         m = self.class.metrics
       rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
@@ -122,16 +183,33 @@ module Streams
     end
 
     def can_provision?(m)
-      m[:overflow_nodes] < self.class.max_overflow_nodes
+      return false if m[:overflow_nodes] >= self.class.max_overflow_nodes
+      return false unless self.class.within_budget?(m[:overflow_nodes] + 1)
+      return false if self.class.kind == "cloud" && !self.class.allow_cloud?
+
+      true
     end
 
     def provision_overflow!
       case self.class.kind
-      when "cloud" then @provisioner.provision_cloud!
-      else @provisioner.provision_lab!
+      when "cloud"
+        raise NodeProvisioner::Error, "Cloud autoscale disabilitato (STREAM_AUTOSCALE_ALLOW_CLOUD / HCLOUD_TOKEN)" unless self.class.allow_cloud?
+
+        @provisioner.provision_cloud!
+      else
+        @provisioner.provision_lab!
       end
     end
 
+    def safe_scale_in!(node)
+      @provisioner.drain!(node) unless node.status == "draining"
+      node.reload
+      raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
+      raise NodeProvisioner::Error, "idle insufficiente" unless idle_long_enough?(node)
+
+      @provisioner.decommission!(node)
+    end
+
     def scale_in_candidates
       StreamNode.where.not(slug: NodeRegistry::HOME_SLUG)
         .where(status: %w[ready draining])

backend/app/views/admin/stream_nodes/index.html.erb

@@ -16,6 +16,15 @@
         max: @autoscale_metrics[:max_overflow_nodes],
         kind: @autoscale_metrics[:kind]
       ) %>
+  · <%= t(
+        "admin.stream_nodes.autoscale_budget",
+        eur: @autoscale_metrics[:estimated_monthly_eur],
+        budget: @autoscale_metrics[:monthly_budget_eur],
+        ok: (@autoscale_metrics[:within_budget] ? "OK" : "OVER")
+      ) %>
+  <% if @autoscale_metrics[:kill_switch] %>
+    · <strong><%= t("admin.stream_nodes.kill_switch_active") %></strong>
+  <% end %>
 </p>
 
 <p>
@@ -26,6 +35,12 @@
   <% else %>
     <span class="admin-muted"><%= t("admin.stream_nodes.hetzner_token_missing") %></span>
   <% end %>
+  <% if @autoscale_metrics[:kill_switch] %>
+    <%= button_to t("admin.stream_nodes.clear_kill_switch"), clear_kill_switch_admin_stream_nodes_path, method: :delete, class: "admin-btn admin-btn-secondary" %>
+  <% else %>
+    <%= button_to t("admin.stream_nodes.engage_kill_switch"), kill_switch_admin_stream_nodes_path, method: :post, class: "admin-btn admin-btn-danger",
+          form: { data: { confirm: t("admin.stream_nodes.kill_switch_confirm") } } %>
+  <% end %>
 </p>
 
 <table class="admin-table">