Alza soft capacity fase A e aggiunge quiet hours CPX notturne.
Home a 4 e cloud a 6 con MAX_NODES=12 (~76 soft); di notte (02–07 Europe/Rome) niente scale-out/warm-spare e sweeper che chiude i CPX idle. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -71,6 +71,10 @@ module Streams
|
||||
estimated_monthly_eur(overflow_count) <= monthly_budget_eur
|
||||
end
|
||||
|
||||
def quiet_hours?
|
||||
QuietHours.active?
|
||||
end
|
||||
|
||||
def reconcile!(provisioner: nil)
|
||||
return Result.new(skipped: true, actions: [], metrics: metrics) unless enabled?
|
||||
|
||||
@@ -106,7 +110,8 @@ module Streams
|
||||
allow_cloud: allow_cloud?,
|
||||
estimated_monthly_eur: estimated_monthly_eur(overflow.size),
|
||||
monthly_budget_eur: monthly_budget_eur,
|
||||
within_budget: within_budget?(overflow.size)
|
||||
within_budget: within_budget?(overflow.size),
|
||||
quiet_hours: quiet_hours?
|
||||
}
|
||||
end
|
||||
|
||||
@@ -201,6 +206,7 @@ module Streams
|
||||
end
|
||||
|
||||
def warm_spare_desired?(m)
|
||||
return false if self.class.quiet_hours?
|
||||
return false if self.class.warm_spare_min <= 0
|
||||
|
||||
need_capacity?(m) || overflow_in_use?
|
||||
@@ -211,6 +217,7 @@ module Streams
|
||||
end
|
||||
|
||||
def can_provision?(m)
|
||||
return false if self.class.quiet_hours?
|
||||
return false if m[:overflow_nodes] >= self.class.max_overflow_nodes
|
||||
return false unless self.class.within_budget?(m[:overflow_nodes] + 1)
|
||||
return false if self.class.kind == "cloud" && !self.class.allow_cloud?
|
||||
@@ -249,6 +256,9 @@ module Streams
|
||||
end
|
||||
|
||||
def idle_long_enough?(node)
|
||||
# Di notte chiudi subito i nodi idle (niente attesa IDLE_MINUTES).
|
||||
return true if self.class.quiet_hours?
|
||||
|
||||
idle_since(node) <= self.class.idle_minutes.minutes.ago
|
||||
end
|
||||
|
||||
@@ -258,6 +268,7 @@ module Streams
|
||||
end
|
||||
|
||||
def keep_as_warm_spare?(node)
|
||||
return false if self.class.quiet_hours?
|
||||
return false unless warm_spare_desired?(self.class.metrics)
|
||||
|
||||
spares = StreamNode.ready.where.not(slug: NodeRegistry::HOME_SLUG).select { |n| n.active_publishers.zero? }
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Streams
|
||||
# Durante quiet hours chiude CPX cloud idle; se hanno sessioni attive solo alert Ops.
|
||||
class NightCloudSweeper
|
||||
Result = Struct.new(:skipped, :actions, :error, keyword_init: true)
|
||||
|
||||
class << self
|
||||
def enabled?
|
||||
ENV.fetch("STREAM_NIGHT_SWEEP_ENABLED", "1") == "1"
|
||||
end
|
||||
|
||||
def sweep!(provisioner: nil)
|
||||
new(provisioner: provisioner).sweep!
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(provisioner: nil)
|
||||
@provisioner = provisioner || NodeProvisioner.new
|
||||
end
|
||||
|
||||
def sweep!
|
||||
unless self.class.enabled?
|
||||
return Result.new(skipped: true, actions: [], error: "disabled")
|
||||
end
|
||||
unless QuietHours.active?
|
||||
return Result.new(skipped: true, actions: [], error: "outside_quiet_hours")
|
||||
end
|
||||
|
||||
actions = []
|
||||
cloud_nodes.find_each do |node|
|
||||
if node.occupying_sessions.exists?
|
||||
alert_active_night_node!(node)
|
||||
actions << :"alert_active_#{node.slug}"
|
||||
next
|
||||
end
|
||||
|
||||
close_idle_node!(node)
|
||||
actions << :"decommission_#{node.slug}"
|
||||
rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
|
||||
Rails.logger.warn("[Streams::NightCloudSweeper] #{node.slug}: #{e.message}")
|
||||
actions << :"error_#{node.slug}"
|
||||
end
|
||||
|
||||
Rails.logger.info("[Streams::NightCloudSweeper] actions=#{actions.inspect}")
|
||||
Result.new(skipped: false, actions: actions)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def cloud_nodes
|
||||
StreamNode.where(role: "cloud").where(status: %w[ready draining provisioning])
|
||||
end
|
||||
|
||||
def alert_active_night_node!(node)
|
||||
Ops::IncidentRecorder.record(
|
||||
{
|
||||
kind: "stream_overflow",
|
||||
severity: "warning",
|
||||
title: "CPX attivo di notte — verificare evento",
|
||||
message: "Nodo #{node.slug} ha #{node.active_publishers} sessione/i in quiet hours (#{QuietHours.range_config} #{QuietHours.time_zone_name})",
|
||||
metadata: {
|
||||
"slug" => node.slug,
|
||||
"session_ids" => node.occupying_sessions.pluck(:id),
|
||||
"quiet_hours" => QuietHours.range_config
|
||||
},
|
||||
fingerprint: "stream_overflow:night_active:#{node.slug}"
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def close_idle_node!(node)
|
||||
@provisioner.drain!(node) unless node.status == "draining"
|
||||
node.reload
|
||||
raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
|
||||
|
||||
slug = node.slug
|
||||
@provisioner.decommission!(node)
|
||||
Rails.logger.warn("[Streams::NightCloudSweeper] decommissioned idle cloud node #{slug} during quiet hours")
|
||||
Ops::IncidentRecorder.resolve(fingerprint: "stream_overflow:night_active:#{slug}")
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,59 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Streams
|
||||
# Finestra notturna (default 02:00–07:00 Europe/Rome) in cui non devono restare CPX idle.
|
||||
module QuietHours
|
||||
DEFAULT_RANGE = "02:00-07:00"
|
||||
DEFAULT_TZ = "Europe/Rome"
|
||||
|
||||
module_function
|
||||
|
||||
def range_config
|
||||
ENV.fetch("STREAM_AUTOSCALE_QUIET_HOURS", DEFAULT_RANGE).to_s.strip
|
||||
end
|
||||
|
||||
def time_zone_name
|
||||
ENV.fetch("STREAM_AUTOSCALE_QUIET_TZ", DEFAULT_TZ)
|
||||
end
|
||||
|
||||
def configured?
|
||||
range_config.present? && range_config != "off" && range_config != "0"
|
||||
end
|
||||
|
||||
def active?(now: Time.current)
|
||||
return false unless configured?
|
||||
|
||||
zone = ActiveSupport::TimeZone[time_zone_name] || Time.find_zone!(time_zone_name)
|
||||
local = now.in_time_zone(zone)
|
||||
start_min, end_min = parse_range(range_config)
|
||||
return false if start_min.nil? || end_min.nil?
|
||||
|
||||
current = local.hour * 60 + local.min
|
||||
if start_min <= end_min
|
||||
current >= start_min && current < end_min
|
||||
else
|
||||
# es. 22:00-06:00
|
||||
current >= start_min || current < end_min
|
||||
end
|
||||
rescue ArgumentError => e
|
||||
Rails.logger.warn("[Streams::QuietHours] invalid config: #{e.message}")
|
||||
false
|
||||
end
|
||||
|
||||
def parse_range(raw)
|
||||
start_s, end_s = raw.split("-", 2).map { |p| p.to_s.strip }
|
||||
return [nil, nil] if start_s.blank? || end_s.blank?
|
||||
|
||||
[parse_hhmm(start_s), parse_hhmm(end_s)]
|
||||
end
|
||||
|
||||
def parse_hhmm(value)
|
||||
h, m = value.split(":", 2)
|
||||
hours = Integer(h)
|
||||
mins = Integer(m || 0)
|
||||
raise ArgumentError, "ora fuori range: #{value}" unless hours.between?(0, 23) && mins.between?(0, 59)
|
||||
|
||||
hours * 60 + mins
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user