84 lines
2.7 KiB
Ruby
84 lines
2.7 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module Streams
|
|
# Durante quiet hours chiude CPX cloud idle; se hanno sessioni attive solo alert Ops.
|
|
class NightCloudSweeper
|
|
Result = Struct.new(:skipped, :actions, :error, keyword_init: true)
|
|
|
|
class << self
|
|
def enabled?
|
|
ENV.fetch("STREAM_NIGHT_SWEEP_ENABLED", "1") == "1"
|
|
end
|
|
|
|
def sweep!(provisioner: nil)
|
|
new(provisioner: provisioner).sweep!
|
|
end
|
|
end
|
|
|
|
def initialize(provisioner: nil)
|
|
@provisioner = provisioner || NodeProvisioner.new
|
|
end
|
|
|
|
def sweep!
|
|
unless self.class.enabled?
|
|
return Result.new(skipped: true, actions: [], error: "disabled")
|
|
end
|
|
unless QuietHours.active?
|
|
return Result.new(skipped: true, actions: [], error: "outside_quiet_hours")
|
|
end
|
|
|
|
actions = []
|
|
cloud_nodes.find_each do |node|
|
|
if node.occupying_sessions.exists?
|
|
alert_active_night_node!(node)
|
|
actions << :"alert_active_#{node.slug}"
|
|
next
|
|
end
|
|
|
|
close_idle_node!(node)
|
|
actions << :"decommission_#{node.slug}"
|
|
rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e
|
|
Rails.logger.warn("[Streams::NightCloudSweeper] #{node.slug}: #{e.message}")
|
|
actions << :"error_#{node.slug}"
|
|
end
|
|
|
|
Rails.logger.info("[Streams::NightCloudSweeper] actions=#{actions.inspect}")
|
|
Result.new(skipped: false, actions: actions)
|
|
end
|
|
|
|
private
|
|
|
|
def cloud_nodes
|
|
StreamNode.where(role: "cloud").where(status: %w[ready draining provisioning])
|
|
end
|
|
|
|
def alert_active_night_node!(node)
|
|
Ops::IncidentRecorder.record(
|
|
{
|
|
kind: "stream_overflow",
|
|
severity: "warning",
|
|
title: "CPX attivo di notte — verificare evento",
|
|
message: "Nodo #{node.slug} ha #{node.active_publishers} sessione/i in quiet hours (#{QuietHours.range_config} #{QuietHours.time_zone_name})",
|
|
metadata: {
|
|
"slug" => node.slug,
|
|
"session_ids" => node.occupying_sessions.pluck(:id),
|
|
"quiet_hours" => QuietHours.range_config
|
|
},
|
|
fingerprint: "stream_overflow:night_active:#{node.slug}"
|
|
}
|
|
)
|
|
end
|
|
|
|
def close_idle_node!(node)
|
|
@provisioner.drain!(node) unless node.status == "draining"
|
|
node.reload
|
|
raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists?
|
|
|
|
slug = node.slug
|
|
@provisioner.decommission!(node)
|
|
Rails.logger.warn("[Streams::NightCloudSweeper] decommissioned idle cloud node #{slug} during quiet hours")
|
|
Ops::IncidentRecorder.resolve(fingerprint: "stream_overflow:night_active:#{slug}")
|
|
end
|
|
end
|
|
end
|