Alza soft capacity fase A e aggiunge quiet hours CPX notturne.
Home a 4 e cloud a 6 con MAX_NODES=12 (~76 soft); di notte (02–07 Europe/Rome) niente scale-out/warm-spare e sweeper che chiude i CPX idle. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Streams
|
||||
# Durante quiet hours chiude CPX cloud idle; se hanno sessioni attive solo alert Ops.
|
||||
class NightCloudSweeper
|
||||
Result = Struct.new(:skipped, :actions, :error, keyword_init: true)
|
||||
|
||||
class << self
|
||||
def enabled?
|
||||
ENV.fetch("STREAM_NIGHT_SWEEP_ENABLED", "1") == "1"
|
||||
end
|
||||
|
||||
def sweep!(provisioner: nil)
|
||||
new(provisioner: provisioner).sweep!
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(provisioner: nil)
|
||||
@provisioner = provisioner || NodeProvisioner.new
|
||||
end
|
||||
|
||||
def sweep!
|
||||
unless self.class.enabled?
|
||||
return Result.new(skipped: true, actions: [], error: "disabled")
|
||||
end
|
||||
unless QuietHours.active?
|
||||
return Result.new(skipped: true, actions: [], error: "outside_quiet_hours")
|
||||
end
|
||||
|
||||
actions = []
|
||||
cloud_nodes.find_each do |node|
|
||||
if node.occupying_sessions.exists?
|
||||
alert_active_night_node!(node)
|
||||
actions << :"alert_active_#{node.slug}"
|
||||
next
|
||||
end
|
||||
|
||||
close_idle_node!(node)
|
||||
actions << :"decommission_#{node.slug}"
|
||||
rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
|
||||
Rails.logger.warn("[Streams::NightCloudSweeper] #{node.slug}: #{e.message}")
|
||||
actions << :"error_#{node.slug}"
|
||||
end
|
||||
|
||||
Rails.logger.info("[Streams::NightCloudSweeper] actions=#{actions.inspect}")
|
||||
Result.new(skipped: false, actions: actions)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def cloud_nodes
|
||||
StreamNode.where(role: "cloud").where(status: %w[ready draining provisioning])
|
||||
end
|
||||
|
||||
def alert_active_night_node!(node)
|
||||
Ops::IncidentRecorder.record(
|
||||
{
|
||||
kind: "stream_overflow",
|
||||
severity: "warning",
|
||||
title: "CPX attivo di notte — verificare evento",
|
||||
message: "Nodo #{node.slug} ha #{node.active_publishers} sessione/i in quiet hours (#{QuietHours.range_config} #{QuietHours.time_zone_name})",
|
||||
metadata: {
|
||||
"slug" => node.slug,
|
||||
"session_ids" => node.occupying_sessions.pluck(:id),
|
||||
"quiet_hours" => QuietHours.range_config
|
||||
},
|
||||
fingerprint: "stream_overflow:night_active:#{node.slug}"
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
def close_idle_node!(node)
|
||||
@provisioner.drain!(node) unless node.status == "draining"
|
||||
node.reload
|
||||
raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
|
||||
|
||||
slug = node.slug
|
||||
@provisioner.decommission!(node)
|
||||
Rails.logger.warn("[Streams::NightCloudSweeper] decommissioned idle cloud node #{slug} during quiet hours")
|
||||
Ops::IncidentRecorder.resolve(fingerprint: "stream_overflow:night_active:#{slug}")
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user