Completa il hardening autoscaler (Fase 4): budget, kill-switch e runbook.

Drain sicuro in scale-in, alert Ops su overflow e controlli admin senza abilitare il deploy in produzione.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/app/controllers/admin/stream_nodes_controller.rb

@@ -44,6 +44,16 @@ module Admin
       redirect_to admin_stream_nodes_path, alert: e.message
     end
 
+    def kill_switch
+      Streams::Autoscaler.engage_kill_switch!
+      redirect_to admin_stream_nodes_path, notice: t("admin.flash.autoscale_kill_on")
+    end
+
+    def clear_kill_switch
+      Streams::Autoscaler.clear_kill_switch!
+      redirect_to admin_stream_nodes_path, notice: t("admin.flash.autoscale_kill_off")
+    end
+
     private
 
     def lab_dns_snippet

backend/app/models/ops/incident.rb

@@ -4,7 +4,7 @@ module Ops
 
     KINDS = %w[
       disk_space recordings_size service_down http_public http_rails rails_latency
-      sidekiq_stale sidekiq_dead log_pattern garage_storage
+      sidekiq_stale sidekiq_dead log_pattern garage_storage stream_overflow
     ].freeze
     SEVERITIES = %w[critical warning info].freeze
     STATUSES = %w[open acknowledged resolved].freeze

backend/app/services/ops/health_checks.rb

@@ -44,7 +44,8 @@ module Ops
         check_sidekiq_heartbeat,
         check_sidekiq_dead,
         check_http_rails,
-        check_rails_latency
+        check_rails_latency,
+        check_stream_overflow
       ]
       findings << check_http_public if public_check_due?
       findings
@@ -161,6 +162,58 @@ module Ops
       fail_finding("garage_storage", "warning", "garage_storage:head", "Garage storage non raggiungibile", e.message)
     end
 
+    def check_stream_overflow
+      return ok_finding("stream_overflow:skip", "Nodi stream non migrati") unless ActiveRecord::Base.connection.data_source_exists?("stream_nodes")
+
+      orphan_hours = ENV.fetch("STREAM_OVERFLOW_ORPHAN_HOURS", "3").to_i
+      orphans = StreamNode.where.not(slug: "home").where(status: %w[ready draining]).select do |n|
+        n.active_publishers.zero? && n.created_at < orphan_hours.hours.ago
+      end
+      metrics = Streams::Autoscaler.metrics
+      over_budget = !metrics[:within_budget]
+      at_max = metrics[:overflow_nodes] >= metrics[:max_overflow_nodes] && metrics[:free_slots] <= metrics[:soft_free_slots]
+
+      if orphans.any?
+        return Finding.new(
+          kind: "stream_overflow",
+          severity: "warning",
+          healthy: false,
+          title: "Nodi stream overflow idle",
+          message: "#{orphans.size} nodo/i idle da >#{orphan_hours}h: #{orphans.map(&:slug).join(', ')}",
+          metadata: { "slugs" => orphans.map(&:slug) },
+          fingerprint: "stream_overflow:orphan_idle"
+        )
+      end
+
+      if over_budget
+        return Finding.new(
+          kind: "stream_overflow",
+          severity: "warning",
+          healthy: false,
+          title: "Budget overflow streaming",
+          message: "Stima €#{metrics[:estimated_monthly_eur]}/mese > budget €#{metrics[:monthly_budget_eur]}",
+          metadata: metrics.transform_keys(&:to_s),
+          fingerprint: "stream_overflow:budget"
+        )
+      end
+
+      if at_max
+        return Finding.new(
+          kind: "stream_overflow",
+          severity: "warning",
+          healthy: false,
+          title: "Capacità stream al massimo",
+          message: "overflow=#{metrics[:overflow_nodes]}/#{metrics[:max_overflow_nodes]} free_slots=#{metrics[:free_slots]}",
+          metadata: metrics.transform_keys(&:to_s),
+          fingerprint: "stream_overflow:at_max"
+        )
+      end
+
+      ok_finding("stream_overflow:ok", "Overflow streaming OK")
+    rescue StandardError => e
+      fail_finding("stream_overflow", "warning", "stream_overflow:error", "Check overflow fallito", e.message)
+    end
+
     def check_sidekiq_heartbeat
       redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
       last = redis.get(Ops::HealthMonitorJob::HEARTBEAT_KEY).to_i

backend/app/services/streams/autoscaler.rb

@@ -2,15 +2,31 @@
 
 module Streams
   # Scale-out / warm spare / scale-in dei nodi overflow (lab o Hetzner).
-  # Kill-switch: STREAM_AUTOSCALE_ENABLED!=1 → no-op.
+  # Kill-switch: STREAM_AUTOSCALE_ENABLED!=1 OPPURE Redis streams:autoscaler:kill_switch=1.
   class Autoscaler
     Result = Struct.new(:actions, :metrics, :skipped, :error, keyword_init: true)
 
     LOCK_KEY = "streams:autoscaler:lock"
+    KILL_SWITCH_KEY = "streams:autoscaler:kill_switch"
 
     class << self
       def enabled?
-        ENV["STREAM_AUTOSCALE_ENABLED"] == "1"
+        return false if kill_switch_engaged?
+        return false unless ENV["STREAM_AUTOSCALE_ENABLED"] == "1"
+
+        true
+      end
+
+      def kill_switch_engaged?
+        redis_get(KILL_SWITCH_KEY) == "1"
+      end
+
+      def engage_kill_switch!
+        redis_set(KILL_SWITCH_KEY, "1")
+      end
+
+      def clear_kill_switch!
+        redis_del(KILL_SWITCH_KEY)
       end
 
       def soft_free_slots
@@ -33,6 +49,28 @@ module Streams
         ENV.fetch("STREAM_AUTOSCALE_KIND", "lab") # lab|cloud
       end
 
+      def allow_cloud?
+        ENV["STREAM_AUTOSCALE_ALLOW_CLOUD"] == "1" && ENV["HCLOUD_TOKEN"].present?
+      end
+
+      def node_eur_per_hour
+        ENV.fetch("STREAM_AUTOSCALE_NODE_EUR_PER_HOUR", "0.015").to_f
+      end
+
+      def monthly_budget_eur
+        ENV.fetch("STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR", "40").to_f
+      end
+
+      def estimated_monthly_eur(overflow_count = nil)
+        count = overflow_count || metrics[:overflow_nodes]
+        # Worst case: nodi sempre accesi 24/7
+        (count * node_eur_per_hour * 24 * 30).round(2)
+      end
+
+      def within_budget?(overflow_count = nil)
+        estimated_monthly_eur(overflow_count) <= monthly_budget_eur
+      end
+
       def reconcile!(provisioner: nil)
         return Result.new(skipped: true, actions: [], metrics: metrics) unless enabled?
 
@@ -62,13 +100,33 @@ module Streams
           warm_spare_min: warm_spare_min,
           max_overflow_nodes: max_overflow_nodes,
           enabled: enabled?,
-          kind: kind
+          env_enabled: ENV["STREAM_AUTOSCALE_ENABLED"] == "1",
+          kill_switch: kill_switch_engaged?,
+          kind: kind,
+          allow_cloud: allow_cloud?,
+          estimated_monthly_eur: estimated_monthly_eur(overflow.size),
+          monthly_budget_eur: monthly_budget_eur,
+          within_budget: within_budget?(overflow.size)
         }
       end
 
       def worker_id
         ENV.fetch("HOSTNAME", "autoscaler")
       end
+
+      def redis_get(key)
+        Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")).get(key)
+      rescue Redis::BaseError
+        nil
+      end
+
+      def redis_set(key, value)
+        Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")).set(key, value)
+      end
+
+      def redis_del(key)
+        Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")).del(key)
+      end
     end
 
     def initialize(provisioner: nil)
@@ -83,6 +141,9 @@ module Streams
         provision_overflow!
         actions << :scale_out
         m = self.class.metrics
+      elsif need_capacity?(m) && !can_provision?(m)
+        actions << :blocked_capacity
+        Rails.logger.warn("[Streams::Autoscaler] capacity needed but blocked metrics=#{m.inspect}")
       end
 
       if warm_spare_desired?(m) && m[:spare_ready] < self.class.warm_spare_min && can_provision?(m)
@@ -94,7 +155,7 @@ module Streams
       scale_in_candidates.each do |node|
         next if keep_as_warm_spare?(node)
 
-        @provisioner.decommission!(node)
+        safe_scale_in!(node)
         actions << :"scale_in_#{node.slug}"
         m = self.class.metrics
       rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
@@ -122,16 +183,33 @@ module Streams
     end
 
     def can_provision?(m)
-      m[:overflow_nodes] < self.class.max_overflow_nodes
+      return false if m[:overflow_nodes] >= self.class.max_overflow_nodes
+      return false unless self.class.within_budget?(m[:overflow_nodes] + 1)
+      return false if self.class.kind == "cloud" && !self.class.allow_cloud?
+
+      true
     end
 
     def provision_overflow!
       case self.class.kind
-      when "cloud" then @provisioner.provision_cloud!
-      else @provisioner.provision_lab!
+      when "cloud"
+        raise NodeProvisioner::Error, "Cloud autoscale disabilitato (STREAM_AUTOSCALE_ALLOW_CLOUD / HCLOUD_TOKEN)" unless self.class.allow_cloud?
+
+        @provisioner.provision_cloud!
+      else
+        @provisioner.provision_lab!
       end
     end
 
+    def safe_scale_in!(node)
+      @provisioner.drain!(node) unless node.status == "draining"
+      node.reload
+      raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
+      raise NodeProvisioner::Error, "idle insufficiente" unless idle_long_enough?(node)
+
+      @provisioner.decommission!(node)
+    end
+
     def scale_in_candidates
       StreamNode.where.not(slug: NodeRegistry::HOME_SLUG)
         .where(status: %w[ready draining])

backend/app/views/admin/stream_nodes/index.html.erb

@@ -16,6 +16,15 @@
         max: @autoscale_metrics[:max_overflow_nodes],
         kind: @autoscale_metrics[:kind]
       ) %>
+  · <%= t(
+        "admin.stream_nodes.autoscale_budget",
+        eur: @autoscale_metrics[:estimated_monthly_eur],
+        budget: @autoscale_metrics[:monthly_budget_eur],
+        ok: (@autoscale_metrics[:within_budget] ? "OK" : "OVER")
+      ) %>
+  <% if @autoscale_metrics[:kill_switch] %>
+    · <strong><%= t("admin.stream_nodes.kill_switch_active") %></strong>
+  <% end %>
 </p>
 
 <p>
@@ -26,6 +35,12 @@
   <% else %>
     <span class="admin-muted"><%= t("admin.stream_nodes.hetzner_token_missing") %></span>
   <% end %>
+  <% if @autoscale_metrics[:kill_switch] %>
+    <%= button_to t("admin.stream_nodes.clear_kill_switch"), clear_kill_switch_admin_stream_nodes_path, method: :delete, class: "admin-btn admin-btn-secondary" %>
+  <% else %>
+    <%= button_to t("admin.stream_nodes.engage_kill_switch"), kill_switch_admin_stream_nodes_path, method: :post, class: "admin-btn admin-btn-danger",
+          form: { data: { confirm: t("admin.stream_nodes.kill_switch_confirm") } } %>
+  <% end %>
 </p>
 
 <table class="admin-table">

backend/config/locales/admin.de.yml

@@ -30,6 +30,8 @@ de:
       stream_node_created: "Lab-Knoten %{slug} bereitgestellt."
       stream_node_destroyed: "Knoten %{slug} entfernt."
       stream_node_draining: "Knoten %{slug} im Drain-Modus."
+      autoscale_kill_on: "Autoscaler-Kill-Switch aktiv. Kein automatisches Scale-out."
+      autoscale_kill_off: "Autoscaler-Kill-Switch aus (STREAM_AUTOSCALE_ENABLED=1 weiterhin nötig)."
       comped_granted: "Kostenloses Abonnement %{plan} für %{club} aktiviert."
       comped_revoked: "Kostenloses Abonnement für %{club} widerrufen."
       session_already_terminated: "Sitzung bereits beendet (%{status})."
@@ -118,6 +120,12 @@ de:
     stream_nodes:
       title: Stream-Knoten
       providers: "Cloud-Provider: %{cloud} · DNS-Provider: %{dns}"
+      autoscale: "Autoscaler %{enabled} · free_slots=%{free} (soft≤%{soft}) · spare=%{spare}/%{warm} · overflow=%{overflow}/%{max} · kind=%{kind}"
+      autoscale_budget: "Budget €%{eur}/€%{budget} (%{ok})"
+      kill_switch_active: "KILL-SWITCH AKTIV"
+      engage_kill_switch: "Kill-switch ON"
+      clear_kill_switch: "Kill-switch OFF"
+      kill_switch_confirm: "Autoscaler sofort blockieren?"
       provision_lab: Lab-Knoten bereitstellen
       drain: Drain
       destroy: Löschen

backend/config/locales/admin.en.yml

@@ -30,6 +30,8 @@ en:
       stream_node_created: "Lab node %{slug} provisioned."
       stream_node_destroyed: "Node %{slug} removed."
       stream_node_draining: "Node %{slug} is draining (no new sessions)."
+      autoscale_kill_on: "Autoscaler kill-switch engaged. No automatic scale-out."
+      autoscale_kill_off: "Autoscaler kill-switch cleared (still needs STREAM_AUTOSCALE_ENABLED=1)."
       comped_granted: "%{plan} complimentary subscription activated for %{club}."
       comped_revoked: "Complimentary subscription revoked for %{club}."
       session_already_terminated: "Session already ended (%{status})."
@@ -119,6 +121,11 @@ en:
       title: Streaming nodes
       providers: "Cloud provider: %{cloud} · DNS provider: %{dns}"
       autoscale: "Autoscaler %{enabled} · free_slots=%{free} (soft≤%{soft}) · spare=%{spare}/%{warm} · overflow=%{overflow}/%{max} · kind=%{kind}"
+      autoscale_budget: "budget €%{eur}/€%{budget} (%{ok})"
+      kill_switch_active: "KILL-SWITCH ACTIVE"
+      engage_kill_switch: "Kill-switch ON (block autoscaler)"
+      clear_kill_switch: "Kill-switch OFF"
+      kill_switch_confirm: "Immediately block the autoscaler? Existing nodes stay up."
       provision_lab: Provision lab node
       provision_cloud: Provision Hetzner node
       provision_cloud_confirm: "Create a Hetzner Cloud server + DNS record on mltv-stream.net? Billing applies until destroyed."

backend/config/locales/admin.es.yml

@@ -30,6 +30,8 @@ es:
       stream_node_created: "Nodo lab %{slug} provisionado."
       stream_node_destroyed: "Nodo %{slug} eliminado."
       stream_node_draining: "Nodo %{slug} en drain."
+      autoscale_kill_on: "Kill-switch del autoscaler activado. Sin scale-out automático."
+      autoscale_kill_off: "Kill-switch del autoscaler desactivado (hace falta STREAM_AUTOSCALE_ENABLED=1)."
       comped_granted: "Suscripción de cortesía %{plan} activada para %{club}."
       comped_revoked: "Suscripción de cortesía revocada para %{club}."
       session_already_terminated: "La sesión ya ha finalizado (%{status})."
@@ -118,6 +120,12 @@ es:
     stream_nodes:
       title: Nodos streaming
       providers: "Cloud provider: %{cloud} · DNS provider: %{dns}"
+      autoscale: "Autoscaler %{enabled} · free_slots=%{free} (soft≤%{soft}) · spare=%{spare}/%{warm} · overflow=%{overflow}/%{max} · kind=%{kind}"
+      autoscale_budget: "presupuesto €%{eur}/€%{budget} (%{ok})"
+      kill_switch_active: "KILL-SWITCH ACTIVO"
+      engage_kill_switch: "Kill-switch ON"
+      clear_kill_switch: "Kill-switch OFF"
+      kill_switch_confirm: "¿Bloquear el autoscaler inmediatamente?"
       provision_lab: Provisionar nodo lab
       drain: Drain
       destroy: Eliminar

backend/config/locales/admin.fr.yml

@@ -30,6 +30,8 @@ fr:
       stream_node_created: "Nœud lab %{slug} provisionné."
       stream_node_destroyed: "Nœud %{slug} supprimé."
       stream_node_draining: "Nœud %{slug} en drain."
+      autoscale_kill_on: "Kill-switch autoscaler activé. Pas de scale-out automatique."
+      autoscale_kill_off: "Kill-switch autoscaler désactivé (nécessite aussi STREAM_AUTOSCALE_ENABLED=1)."
       comped_granted: "Abonnement offert %{plan} activé pour %{club}."
       comped_revoked: "Abonnement offert révoqué pour %{club}."
       session_already_terminated: "Session déjà terminée (%{status})."
@@ -118,6 +120,12 @@ fr:
     stream_nodes:
       title: Nœuds streaming
       providers: "Cloud provider: %{cloud} · DNS provider: %{dns}"
+      autoscale: "Autoscaler %{enabled} · free_slots=%{free} (soft≤%{soft}) · spare=%{spare}/%{warm} · overflow=%{overflow}/%{max} · kind=%{kind}"
+      autoscale_budget: "budget €%{eur}/€%{budget} (%{ok})"
+      kill_switch_active: "KILL-SWITCH ACTIF"
+      engage_kill_switch: "Kill-switch ON"
+      clear_kill_switch: "Kill-switch OFF"
+      kill_switch_confirm: "Bloquer immédiatement l'autoscaler ?"
       provision_lab: Provisionner un nœud lab
       drain: Drain
       destroy: Supprimer

backend/config/locales/admin.it.yml

@@ -30,6 +30,8 @@ it:
       stream_node_created: "Nodo lab %{slug} provisionato."
       stream_node_destroyed: "Nodo %{slug} rimosso."
       stream_node_draining: "Nodo %{slug} in drain (niente nuove sessioni)."
+      autoscale_kill_on: "Kill-switch autoscaler attivato. Nessun scale-out automatico."
+      autoscale_kill_off: "Kill-switch autoscaler disattivato (serve comunque STREAM_AUTOSCALE_ENABLED=1)."
       comped_granted: "Abbonamento omaggio %{plan} attivato per %{club}."
       comped_revoked: "Abbonamento omaggio revocato per %{club}."
       session_already_terminated: "Sessione già terminata (%{status})."
@@ -119,6 +121,11 @@ it:
       title: Nodi streaming
       providers: "Cloud provider: %{cloud} · DNS provider: %{dns}"
       autoscale: "Autoscaler %{enabled} · free_slots=%{free} (soft≤%{soft}) · spare=%{spare}/%{warm} · overflow=%{overflow}/%{max} · kind=%{kind}"
+      autoscale_budget: "budget €%{eur}/€%{budget} (%{ok})"
+      kill_switch_active: "KILL-SWITCH ATTIVO"
+      engage_kill_switch: "Kill-switch ON (blocca autoscaler)"
+      clear_kill_switch: "Kill-switch OFF"
+      kill_switch_confirm: "Bloccare immediatamente l'autoscaler? I nodi esistenti restano accesi."
       provision_lab: Provisiona nodo lab
       provision_cloud: Provisiona nodo Hetzner
       provision_cloud_confirm: "Creare un server Hetzner Cloud + record DNS su mltv-stream.net? Verrà addebitato fino allo spegnimento."

backend/config/routes.rb

@@ -114,6 +114,10 @@ Rails.application.routes.draw do
       member do
         post :drain
       end
+      collection do
+        post :kill_switch
+        delete :clear_kill_switch
+      end
     end
     get "youtube/platform", to: "youtube#platform", as: :youtube_platform
   end

backend/spec/services/ops/health_checks_spec.rb

@@ -25,6 +25,7 @@ RSpec.describe Ops::HealthChecks do
       %i[
         check_recordings_size check_postgres check_redis check_mediamtx check_garage
         check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
+        check_stream_overflow
       ].each do |method|
         allow_any_instance_of(described_class).to receive(method).and_return(
           described_class::Finding.new(
@@ -52,6 +53,7 @@ RSpec.describe Ops::HealthChecks do
       %i[
         check_recordings_size check_postgres check_redis check_mediamtx check_garage
         check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
+        check_stream_overflow
       ].each do |method|
         allow_any_instance_of(described_class).to receive(method).and_return(
           described_class::Finding.new(
@@ -65,7 +67,7 @@ RSpec.describe Ops::HealthChecks do
       summary = described_class.new.summary
 
       expect(summary[:status]).to eq("ok")
-      expect(summary[:checks].size).to eq(10)
+      expect(summary[:checks].size).to eq(11)
     end
 
     it "degraded quando la latenza p95 supera la soglia warning" do

backend/spec/services/streams/autoscaler_spec.rb

@@ -15,6 +15,7 @@ RSpec.describe Streams::Autoscaler do
 
   before do
     redis.del(Streams::Autoscaler::LOCK_KEY)
+    redis.del(Streams::Autoscaler::KILL_SWITCH_KEY)
     redis.del(Streams::DnsProviders::Lab::REDIS_KEY)
   end
 
@@ -116,10 +117,55 @@ RSpec.describe Streams::Autoscaler do
       )
 
       provisioner = instance_double(Streams::NodeProvisioner)
+      allow(provisioner).to receive(:drain!)
       expect(provisioner).to receive(:decommission!).with(idle)
 
       result = described_class.reconcile!(provisioner: provisioner)
       expect(result.actions.map(&:to_s)).to include("scale_in_ingest-lab-99")
     end
   end
+
+  it "skips when Redis kill-switch is engaged even if ENV is enabled" do
+    redis.del(described_class::KILL_SWITCH_KEY)
+    with_env("STREAM_AUTOSCALE_ENABLED" => "1") do
+      described_class.engage_kill_switch!
+      expect(described_class.enabled?).to eq(false)
+      result = described_class.reconcile!
+      expect(result.skipped).to eq(true)
+    ensure
+      described_class.clear_kill_switch!
+    end
+  end
+
+  it "blocks scale-out when next node would exceed monthly budget" do
+    with_env(
+      "STREAM_AUTOSCALE_ENABLED" => "1",
+      "STREAM_AUTOSCALE_SOFT_FREE_SLOTS" => "2",
+      "STREAM_AUTOSCALE_WARM_SPARE" => "0",
+      "STREAM_AUTOSCALE_KIND" => "lab",
+      "STREAM_AUTOSCALE_MAX_NODES" => "5",
+      "STREAM_AUTOSCALE_NODE_EUR_PER_HOUR" => "1",
+      "STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR" => "10",
+      "STREAM_CLOUD_PROVIDER" => "local_lab",
+      "STREAM_DNS_PROVIDER" => "lab",
+      "STREAM_NODE_HOME_MAX_PUBLISHERS" => "1",
+      "MEDIAMTX_API_URL" => "http://mtx-home:9997",
+      "MEDIAMTX_RTMP_URL" => "rtmp://home.example:1935",
+      "HLS_PUBLIC_URL" => "https://home.example/hls"
+    ) do
+      home = Streams::NodeRegistry.ensure_home_from_env!
+      user = User.create!(email: "budget@example.com", name: "B", password: "Password123", role: "coach")
+      club = Club.create!(name: "Budget", sport: "volleyball")
+      team = club.teams.create!(name: "T", sport: "volleyball", slug: "budget-t")
+      match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now)
+      StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "live", stream_node: home)
+
+      provisioner = instance_double(Streams::NodeProvisioner)
+      expect(provisioner).not_to receive(:provision_lab!)
+
+      result = described_class.reconcile!(provisioner: provisioner)
+      expect(result.actions).to include(:blocked_capacity)
+      expect(described_class.metrics[:within_budget]).to eq(true)
+    end
+  end
 end