Alza soft capacity fase A e aggiunge quiet hours CPX notturne.

Home a 4 e cloud a 6 con MAX_NODES=12 (~76 soft); di notte (02–07 Europe/Rome) niente scale-out/warm-spare e sweeper che chiude i CPX idle. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-05 09:48:50 +02:00
parent d52434fb2e
commit 7c7b2bf14c
16 changed files with 513 additions and 25 deletions
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Streams
+  class NightCloudSweeperJob
+    include Sidekiq::Job
+
+    sidekiq_options retry: 1, queue: "default"
+
+    INTERVAL_SECS = ENV.fetch("STREAM_NIGHT_SWEEP_INTERVAL_SECS", "900").to_i
+    REDIS_CHAIN_KEY = "streams:night_sweep:chain"
+
+    def self.ensure_chain
+      return unless redis
+      return if redis.get(REDIS_CHAIN_KEY)
+
+      redis.set(REDIS_CHAIN_KEY, "1", ex: INTERVAL_SECS * 2)
+      perform_in(INTERVAL_SECS)
+    end
+
+    def self.redis
+      @redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"))
+    rescue Redis::CannotConnectError
+      nil
+    end
+
+    def perform
+      Streams::NightCloudSweeper.sweep!
+    ensure
+      self.class.redis&.set(REDIS_CHAIN_KEY, "1", ex: INTERVAL_SECS * 2)
+      self.class.perform_in(INTERVAL_SECS)
+    end
+  end
+end
@@ -71,6 +71,10 @@ module Streams
        estimated_monthly_eur(overflow_count) <= monthly_budget_eur
      end

+      def quiet_hours?
+        QuietHours.active?
+      end
+
      def reconcile!(provisioner: nil)
        return Result.new(skipped: true, actions: [], metrics: metrics) unless enabled?

@@ -106,7 +110,8 @@ module Streams
          allow_cloud: allow_cloud?,
          estimated_monthly_eur: estimated_monthly_eur(overflow.size),
          monthly_budget_eur: monthly_budget_eur,
-          within_budget: within_budget?(overflow.size)
+          within_budget: within_budget?(overflow.size),
+          quiet_hours: quiet_hours?
        }
      end

@@ -201,6 +206,7 @@ module Streams
    end

    def warm_spare_desired?(m)
+      return false if self.class.quiet_hours?
      return false if self.class.warm_spare_min <= 0

      need_capacity?(m) || overflow_in_use?
@@ -211,6 +217,7 @@ module Streams
    end

    def can_provision?(m)
+      return false if self.class.quiet_hours?
      return false if m[:overflow_nodes] >= self.class.max_overflow_nodes
      return false unless self.class.within_budget?(m[:overflow_nodes] + 1)
      return false if self.class.kind == "cloud" && !self.class.allow_cloud?
@@ -249,6 +256,9 @@ module Streams
    end

    def idle_long_enough?(node)
+      # Di notte chiudi subito i nodi idle (niente attesa IDLE_MINUTES).
+      return true if self.class.quiet_hours?
+
      idle_since(node) <= self.class.idle_minutes.minutes.ago
    end

@@ -258,6 +268,7 @@ module Streams
    end

    def keep_as_warm_spare?(node)
+      return false if self.class.quiet_hours?
      return false unless warm_spare_desired?(self.class.metrics)

      spares = StreamNode.ready.where.not(slug: NodeRegistry::HOME_SLUG).select { |n| n.active_publishers.zero? }
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Streams
+  # Durante quiet hours chiude CPX cloud idle; se hanno sessioni attive solo alert Ops.
+  class NightCloudSweeper
+    Result = Struct.new(:skipped, :actions, :error, keyword_init: true)
+
+    class << self
+      def enabled?
+        ENV.fetch("STREAM_NIGHT_SWEEP_ENABLED", "1") == "1"
+      end
+
+      def sweep!(provisioner: nil)
+        new(provisioner: provisioner).sweep!
+      end
+    end
+
+    def initialize(provisioner: nil)
+      @provisioner = provisioner || NodeProvisioner.new
+    end
+
+    def sweep!
+      unless self.class.enabled?
+        return Result.new(skipped: true, actions: [], error: "disabled")
+      end
+      unless QuietHours.active?
+        return Result.new(skipped: true, actions: [], error: "outside_quiet_hours")
+      end
+
+      actions = []
+      cloud_nodes.find_each do |node|
+        if node.occupying_sessions.exists?
+          alert_active_night_node!(node)
+          actions << :"alert_active_#{node.slug}"
+          next
+        end
+
+        close_idle_node!(node)
+        actions << :"decommission_#{node.slug}"
+      rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
+        Rails.logger.warn("[Streams::NightCloudSweeper] #{node.slug}: #{e.message}")
+        actions << :"error_#{node.slug}"
+      end
+
+      Rails.logger.info("[Streams::NightCloudSweeper] actions=#{actions.inspect}")
+      Result.new(skipped: false, actions: actions)
+    end
+
+    private
+
+    def cloud_nodes
+      StreamNode.where(role: "cloud").where(status: %w[ready draining provisioning])
+    end
+
+    def alert_active_night_node!(node)
+      Ops::IncidentRecorder.record(
+        {
+          kind: "stream_overflow",
+          severity: "warning",
+          title: "CPX attivo di notte — verificare evento",
+          message: "Nodo #{node.slug} ha #{node.active_publishers} sessione/i in quiet hours (#{QuietHours.range_config} #{QuietHours.time_zone_name})",
+          metadata: {
+            "slug" => node.slug,
+            "session_ids" => node.occupying_sessions.pluck(:id),
+            "quiet_hours" => QuietHours.range_config
+          },
+          fingerprint: "stream_overflow:night_active:#{node.slug}"
+        }
+      )
+    end
+
+    def close_idle_node!(node)
+      @provisioner.drain!(node) unless node.status == "draining"
+      node.reload
+      raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
+
+      slug = node.slug
+      @provisioner.decommission!(node)
+      Rails.logger.warn("[Streams::NightCloudSweeper] decommissioned idle cloud node #{slug} during quiet hours")
+      Ops::IncidentRecorder.resolve(fingerprint: "stream_overflow:night_active:#{slug}")
+    end
+  end
+end
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Streams
+  # Finestra notturna (default 02:00–07:00 Europe/Rome) in cui non devono restare CPX idle.
+  module QuietHours
+    DEFAULT_RANGE = "02:00-07:00"
+    DEFAULT_TZ = "Europe/Rome"
+
+    module_function
+
+    def range_config
+      ENV.fetch("STREAM_AUTOSCALE_QUIET_HOURS", DEFAULT_RANGE).to_s.strip
+    end
+
+    def time_zone_name
+      ENV.fetch("STREAM_AUTOSCALE_QUIET_TZ", DEFAULT_TZ)
+    end
+
+    def configured?
+      range_config.present? && range_config != "off" && range_config != "0"
+    end
+
+    def active?(now: Time.current)
+      return false unless configured?
+
+      zone = ActiveSupport::TimeZone[time_zone_name] || Time.find_zone!(time_zone_name)
+      local = now.in_time_zone(zone)
+      start_min, end_min = parse_range(range_config)
+      return false if start_min.nil? || end_min.nil?
+
+      current = local.hour * 60 + local.min
+      if start_min <= end_min
+        current >= start_min && current < end_min
+      else
+        # es. 22:00-06:00
+        current >= start_min || current < end_min
+      end
+    rescue ArgumentError => e
+      Rails.logger.warn("[Streams::QuietHours] invalid config: #{e.message}")
+      false
+    end
+
+    def parse_range(raw)
+      start_s, end_s = raw.split("-", 2).map { |p| p.to_s.strip }
+      return [nil, nil] if start_s.blank? || end_s.blank?
+
+      [parse_hhmm(start_s), parse_hhmm(end_s)]
+    end
+
+    def parse_hhmm(value)
+      h, m = value.split(":", 2)
+      hours = Integer(h)
+      mins = Integer(m || 0)
+      raise ArgumentError, "ora fuori range: #{value}" unless hours.between?(0, 23) && mins.between?(0, 59)
+
+      hours * 60 + mins
+    end
+  end
+end
@@ -7,6 +7,7 @@ Sidekiq.configure_server do |config|
    StreamPublisherSyncJob.ensure_chain
    Ops::HealthMonitorJob.ensure_chain
    Streams::AutoscalerJob.ensure_chain
+    Streams::NightCloudSweeperJob.ensure_chain
  end
 end

@@ -1,6 +1,12 @@
 # frozen_string_literal: true

 namespace :streams do
+  desc "Chiude CPX cloud idle in quiet hours (02:00–07:00 Europe/Rome); alert se sessioni attive"
+  task night_cloud_sweep: :environment do
+    result = Streams::NightCloudSweeper.sweep!
+    puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}"
+  end
+
  namespace :nodes do
    desc "Assicura il nodo home dagli ENV MediaMTX"
    task ensure_home: :environment do
@@ -19,6 +19,11 @@ RSpec.describe Streams::Autoscaler do
    redis.del(Streams::DnsProviders::Lab::REDIS_KEY)
    StreamSession.where.not(stream_node_id: nil).update_all(stream_node_id: nil, status: "ended", ended_at: Time.current)
    StreamNode.where.not(slug: Streams::NodeRegistry::HOME_SLUG).delete_all
+    ENV["STREAM_AUTOSCALE_QUIET_HOURS"] = "off"
+  end
+
+  after do
+    ENV.delete("STREAM_AUTOSCALE_QUIET_HOURS")
  end

  it "is a no-op when disabled" do
@@ -261,4 +266,34 @@ RSpec.describe Streams::Autoscaler do
      expect(node.reload.status).to eq("ready")
    end
  end
+
+  it "blocks scale-out and warm spare during quiet hours" do
+    with_env(
+      "STREAM_AUTOSCALE_ENABLED" => "1",
+      "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00",
+      "STREAM_AUTOSCALE_SOFT_FREE_SLOTS" => "2",
+      "STREAM_AUTOSCALE_WARM_SPARE" => "1",
+      "STREAM_AUTOSCALE_KIND" => "lab",
+      "STREAM_AUTOSCALE_MAX_NODES" => "5",
+      "STREAM_NODE_HOME_MAX_PUBLISHERS" => "1",
+      "MEDIAMTX_API_URL" => "http://mtx-home:9997",
+      "MEDIAMTX_RTMP_URL" => "rtmp://home.example:1935",
+      "HLS_PUBLIC_URL" => "https://home.example/hls"
+    ) do
+      allow(Streams::QuietHours).to receive(:active?).and_return(true)
+      home = Streams::NodeRegistry.ensure_home_from_env!
+      user = User.create!(email: "qh@example.com", name: "Q", password: "Password123", role: "coach")
+      club = Club.create!(name: "QH", sport: "volleyball")
+      team = club.teams.create!(name: "T", sport: "volleyball", slug: "qh-t")
+      match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now)
+      StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "live", stream_node: home)
+
+      provisioner = instance_double(Streams::NodeProvisioner)
+      expect(provisioner).not_to receive(:provision_lab!)
+
+      result = described_class.reconcile!(provisioner: provisioner)
+      expect(result.actions).to include(:blocked_capacity)
+      expect(result.metrics[:quiet_hours]).to eq(true)
+    end
+  end
 end
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Streams::NightCloudSweeper do
+  def with_env(vars)
+    previous = vars.keys.index_with { |k| ENV[k] }
+    vars.each { |k, v| ENV[k] = v }
+    yield
+  ensure
+    previous.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v }
+  end
+
+  def cloud_node!(slug: "ingest-cloud-01")
+    StreamNode.create!(
+      slug: slug,
+      hostname: "#{slug}.mltv-stream.net",
+      role: "cloud",
+      status: "ready",
+      provider: "hetzner",
+      provider_instance_id: "42",
+      rtmp_base_url: "rtmp://#{slug}:1935",
+      hls_base_url: "https://#{slug}/hls",
+      api_base_url: "http://10.0.0.9:9997",
+      max_publishers: 6,
+      max_relays: 6,
+      metadata: { "public_ip" => "1.2.3.4" }
+    )
+  end
+
+  before do
+    StreamSession.where.not(stream_node_id: nil).update_all(stream_node_id: nil, status: "ended", ended_at: Time.current)
+    StreamNode.where(role: "cloud").delete_all
+  end
+
+  it "skips outside quiet hours" do
+    with_env(
+      "STREAM_NIGHT_SWEEP_ENABLED" => "1",
+      "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00",
+      "STREAM_AUTOSCALE_QUIET_TZ" => "Europe/Rome"
+    ) do
+      allow(Streams::QuietHours).to receive(:active?).and_return(false)
+      cloud_node!
+      result = described_class.sweep!
+      expect(result.skipped).to eq(true)
+      expect(StreamNode.where(role: "cloud").count).to eq(1)
+    end
+  end
+
+  it "decommissions idle cloud nodes during quiet hours" do
+    with_env("STREAM_NIGHT_SWEEP_ENABLED" => "1") do
+      allow(Streams::QuietHours).to receive(:active?).and_return(true)
+      node = cloud_node!
+      provisioner = instance_double(Streams::NodeProvisioner)
+      allow(provisioner).to receive(:drain!) do |n|
+        n.update!(status: "draining")
+        n
+      end
+      allow(provisioner).to receive(:decommission!) do |n|
+        n.destroy!
+        true
+      end
+
+      result = described_class.sweep!(provisioner: provisioner)
+      expect(result.skipped).to eq(false)
+      expect(result.actions).to include(:"decommission_#{node.slug}")
+      expect(StreamNode.find_by(id: node.id)).to be_nil
+    end
+  end
+
+  it "alerts and keeps nodes with active sessions" do
+    with_env("STREAM_NIGHT_SWEEP_ENABLED" => "1") do
+      allow(Streams::QuietHours).to receive(:active?).and_return(true)
+      node = cloud_node!
+      user = User.create!(email: "night@example.com", name: "N", password: "Password123", role: "coach")
+      club = Club.create!(name: "Night", sport: "volleyball")
+      team = club.teams.create!(name: "T", sport: "volleyball", slug: "night-t")
+      match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now)
+      StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "live", stream_node: node)
+
+      allow(Ops::IncidentRecorder).to receive(:record)
+      provisioner = instance_double(Streams::NodeProvisioner)
+      expect(provisioner).not_to receive(:decommission!)
+
+      result = described_class.sweep!(provisioner: provisioner)
+      expect(result.actions).to include(:"alert_active_#{node.slug}")
+      expect(Ops::IncidentRecorder).to have_received(:record).with(hash_including(
+        fingerprint: "stream_overflow:night_active:#{node.slug}"
+      ))
+      expect(node.reload.status).to eq("ready")
+    end
+  end
+
+  it "does not touch lab nodes" do
+    with_env("STREAM_NIGHT_SWEEP_ENABLED" => "1") do
+      allow(Streams::QuietHours).to receive(:active?).and_return(true)
+      StreamNode.create!(
+        slug: "ingest-lab-night-01",
+        hostname: "lab-night.local",
+        role: "lab",
+        status: "ready",
+        provider: "local",
+        rtmp_base_url: "rtmp://lab:1935",
+        hls_base_url: "https://lab/hls",
+        api_base_url: "http://lab:9997",
+        max_publishers: 2,
+        max_relays: 2
+      )
+      result = described_class.sweep!
+      expect(result.actions).to eq([])
+      expect(StreamNode.find_by(slug: "ingest-lab-night-01")).to be_present
+    end
+  end
+end
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Streams::QuietHours do
+  def with_env(vars)
+    previous = vars.keys.index_with { |k| ENV[k] }
+    vars.each { |k, v| ENV[k] = v }
+    yield
+  ensure
+    previous.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v }
+  end
+
+  it "is active inside the Rome window" do
+    with_env(
+      "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00",
+      "STREAM_AUTOSCALE_QUIET_TZ" => "Europe/Rome"
+    ) do
+      now = Time.find_zone!("Europe/Rome").local(2026, 9, 5, 3, 30)
+      expect(described_class.active?(now: now)).to eq(true)
+    end
+  end
+
+  it "is inactive outside the window" do
+    with_env(
+      "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00",
+      "STREAM_AUTOSCALE_QUIET_TZ" => "Europe/Rome"
+    ) do
+      now = Time.find_zone!("Europe/Rome").local(2026, 9, 5, 10, 0)
+      expect(described_class.active?(now: now)).to eq(false)
+    end
+  end
+
+  it "can be disabled with off" do
+    with_env("STREAM_AUTOSCALE_QUIET_HOURS" => "off") do
+      now = Time.find_zone!("Europe/Rome").local(2026, 9, 5, 3, 0)
+      expect(described_class.active?(now: now)).to eq(false)
+    end
+  end
+end