Alza soft capacity fase A e aggiunge quiet hours CPX notturne.

Home a 4 e cloud a 6 con MAX_NODES=12 (~76 soft); di notte (02–07 Europe/Rome) niente scale-out/warm-spare e sweeper che chiude i CPX idle.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-09-05 09:48:50 +02:00
co-authored by Cursor
parent d52434fb2e
commit 7c7b2bf14c
16 changed files with 513 additions and 25 deletions
+12 -1
View File
@@ -71,6 +71,10 @@ module Streams
estimated_monthly_eur(overflow_count) <= monthly_budget_eur
end
def quiet_hours?
QuietHours.active?
end
def reconcile!(provisioner: nil)
return Result.new(skipped: true, actions: [], metrics: metrics) unless enabled?
@@ -106,7 +110,8 @@ module Streams
allow_cloud: allow_cloud?,
estimated_monthly_eur: estimated_monthly_eur(overflow.size),
monthly_budget_eur: monthly_budget_eur,
within_budget: within_budget?(overflow.size)
within_budget: within_budget?(overflow.size),
quiet_hours: quiet_hours?
}
end
@@ -201,6 +206,7 @@ module Streams
end
def warm_spare_desired?(m)
return false if self.class.quiet_hours?
return false if self.class.warm_spare_min <= 0
need_capacity?(m) || overflow_in_use?
@@ -211,6 +217,7 @@ module Streams
end
def can_provision?(m)
return false if self.class.quiet_hours?
return false if m[:overflow_nodes] >= self.class.max_overflow_nodes
return false unless self.class.within_budget?(m[:overflow_nodes] + 1)
return false if self.class.kind == "cloud" && !self.class.allow_cloud?
@@ -249,6 +256,9 @@ module Streams
end
def idle_long_enough?(node)
# Di notte chiudi subito i nodi idle (niente attesa IDLE_MINUTES).
return true if self.class.quiet_hours?
idle_since(node) <= self.class.idle_minutes.minutes.ago
end
@@ -258,6 +268,7 @@ module Streams
end
def keep_as_warm_spare?(node)
return false if self.class.quiet_hours?
return false unless warm_spare_desired?(self.class.metrics)
spares = StreamNode.ready.where.not(slug: NodeRegistry::HOME_SLUG).select { |n| n.active_publishers.zero? }
@@ -0,0 +1,83 @@
# frozen_string_literal: true
module Streams
# Durante quiet hours chiude CPX cloud idle; se hanno sessioni attive solo alert Ops.
class NightCloudSweeper
Result = Struct.new(:skipped, :actions, :error, keyword_init: true)
class << self
def enabled?
ENV.fetch("STREAM_NIGHT_SWEEP_ENABLED", "1") == "1"
end
def sweep!(provisioner: nil)
new(provisioner: provisioner).sweep!
end
end
def initialize(provisioner: nil)
@provisioner = provisioner || NodeProvisioner.new
end
def sweep!
unless self.class.enabled?
return Result.new(skipped: true, actions: [], error: "disabled")
end
unless QuietHours.active?
return Result.new(skipped: true, actions: [], error: "outside_quiet_hours")
end
actions = []
cloud_nodes.find_each do |node|
if node.occupying_sessions.exists?
alert_active_night_node!(node)
actions << :"alert_active_#{node.slug}"
next
end
close_idle_node!(node)
actions << :"decommission_#{node.slug}"
rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
Rails.logger.warn("[Streams::NightCloudSweeper] #{node.slug}: #{e.message}")
actions << :"error_#{node.slug}"
end
Rails.logger.info("[Streams::NightCloudSweeper] actions=#{actions.inspect}")
Result.new(skipped: false, actions: actions)
end
private
def cloud_nodes
StreamNode.where(role: "cloud").where(status: %w[ready draining provisioning])
end
def alert_active_night_node!(node)
Ops::IncidentRecorder.record(
{
kind: "stream_overflow",
severity: "warning",
title: "CPX attivo di notte — verificare evento",
message: "Nodo #{node.slug} ha #{node.active_publishers} sessione/i in quiet hours (#{QuietHours.range_config} #{QuietHours.time_zone_name})",
metadata: {
"slug" => node.slug,
"session_ids" => node.occupying_sessions.pluck(:id),
"quiet_hours" => QuietHours.range_config
},
fingerprint: "stream_overflow:night_active:#{node.slug}"
}
)
end
def close_idle_node!(node)
@provisioner.drain!(node) unless node.status == "draining"
node.reload
raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
slug = node.slug
@provisioner.decommission!(node)
Rails.logger.warn("[Streams::NightCloudSweeper] decommissioned idle cloud node #{slug} during quiet hours")
Ops::IncidentRecorder.resolve(fingerprint: "stream_overflow:night_active:#{slug}")
end
end
end
@@ -0,0 +1,59 @@
# frozen_string_literal: true
module Streams
# Finestra notturna (default 02:0007:00 Europe/Rome) in cui non devono restare CPX idle.
module QuietHours
DEFAULT_RANGE = "02:00-07:00"
DEFAULT_TZ = "Europe/Rome"
module_function
def range_config
ENV.fetch("STREAM_AUTOSCALE_QUIET_HOURS", DEFAULT_RANGE).to_s.strip
end
def time_zone_name
ENV.fetch("STREAM_AUTOSCALE_QUIET_TZ", DEFAULT_TZ)
end
def configured?
range_config.present? && range_config != "off" && range_config != "0"
end
def active?(now: Time.current)
return false unless configured?
zone = ActiveSupport::TimeZone[time_zone_name] || Time.find_zone!(time_zone_name)
local = now.in_time_zone(zone)
start_min, end_min = parse_range(range_config)
return false if start_min.nil? || end_min.nil?
current = local.hour * 60 + local.min
if start_min <= end_min
current >= start_min && current < end_min
else
# es. 22:00-06:00
current >= start_min || current < end_min
end
rescue ArgumentError => e
Rails.logger.warn("[Streams::QuietHours] invalid config: #{e.message}")
false
end
def parse_range(raw)
start_s, end_s = raw.split("-", 2).map { |p| p.to_s.strip }
return [nil, nil] if start_s.blank? || end_s.blank?
[parse_hhmm(start_s), parse_hhmm(end_s)]
end
def parse_hhmm(value)
h, m = value.split(":", 2)
hours = Integer(h)
mins = Integer(m || 0)
raise ArgumentError, "ora fuori range: #{value}" unless hours.between?(0, 23) && mins.between?(0, 59)
hours * 60 + mins
end
end
end