Completa il hardening autoscaler (Fase 4): budget, kill-switch e runbook.
Drain sicuro in scale-in, alert Ops su overflow e controlli admin senza abilitare il deploy in produzione. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -25,6 +25,7 @@ RSpec.describe Ops::HealthChecks do
|
||||
%i[
|
||||
check_recordings_size check_postgres check_redis check_mediamtx check_garage
|
||||
check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
|
||||
check_stream_overflow
|
||||
].each do |method|
|
||||
allow_any_instance_of(described_class).to receive(method).and_return(
|
||||
described_class::Finding.new(
|
||||
@@ -52,6 +53,7 @@ RSpec.describe Ops::HealthChecks do
|
||||
%i[
|
||||
check_recordings_size check_postgres check_redis check_mediamtx check_garage
|
||||
check_sidekiq_heartbeat check_sidekiq_dead check_http_rails check_rails_latency
|
||||
check_stream_overflow
|
||||
].each do |method|
|
||||
allow_any_instance_of(described_class).to receive(method).and_return(
|
||||
described_class::Finding.new(
|
||||
@@ -65,7 +67,7 @@ RSpec.describe Ops::HealthChecks do
|
||||
summary = described_class.new.summary
|
||||
|
||||
expect(summary[:status]).to eq("ok")
|
||||
expect(summary[:checks].size).to eq(10)
|
||||
expect(summary[:checks].size).to eq(11)
|
||||
end
|
||||
|
||||
it "degraded quando la latenza p95 supera la soglia warning" do
|
||||
|
||||
@@ -15,6 +15,7 @@ RSpec.describe Streams::Autoscaler do
|
||||
|
||||
before do
|
||||
redis.del(Streams::Autoscaler::LOCK_KEY)
|
||||
redis.del(Streams::Autoscaler::KILL_SWITCH_KEY)
|
||||
redis.del(Streams::DnsProviders::Lab::REDIS_KEY)
|
||||
end
|
||||
|
||||
@@ -116,10 +117,55 @@ RSpec.describe Streams::Autoscaler do
|
||||
)
|
||||
|
||||
provisioner = instance_double(Streams::NodeProvisioner)
|
||||
allow(provisioner).to receive(:drain!)
|
||||
expect(provisioner).to receive(:decommission!).with(idle)
|
||||
|
||||
result = described_class.reconcile!(provisioner: provisioner)
|
||||
expect(result.actions.map(&:to_s)).to include("scale_in_ingest-lab-99")
|
||||
end
|
||||
end
|
||||
|
||||
it "skips when Redis kill-switch is engaged even if ENV is enabled" do
|
||||
redis.del(described_class::KILL_SWITCH_KEY)
|
||||
with_env("STREAM_AUTOSCALE_ENABLED" => "1") do
|
||||
described_class.engage_kill_switch!
|
||||
expect(described_class.enabled?).to eq(false)
|
||||
result = described_class.reconcile!
|
||||
expect(result.skipped).to eq(true)
|
||||
ensure
|
||||
described_class.clear_kill_switch!
|
||||
end
|
||||
end
|
||||
|
||||
it "blocks scale-out when next node would exceed monthly budget" do
|
||||
with_env(
|
||||
"STREAM_AUTOSCALE_ENABLED" => "1",
|
||||
"STREAM_AUTOSCALE_SOFT_FREE_SLOTS" => "2",
|
||||
"STREAM_AUTOSCALE_WARM_SPARE" => "0",
|
||||
"STREAM_AUTOSCALE_KIND" => "lab",
|
||||
"STREAM_AUTOSCALE_MAX_NODES" => "5",
|
||||
"STREAM_AUTOSCALE_NODE_EUR_PER_HOUR" => "1",
|
||||
"STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR" => "10",
|
||||
"STREAM_CLOUD_PROVIDER" => "local_lab",
|
||||
"STREAM_DNS_PROVIDER" => "lab",
|
||||
"STREAM_NODE_HOME_MAX_PUBLISHERS" => "1",
|
||||
"MEDIAMTX_API_URL" => "http://mtx-home:9997",
|
||||
"MEDIAMTX_RTMP_URL" => "rtmp://home.example:1935",
|
||||
"HLS_PUBLIC_URL" => "https://home.example/hls"
|
||||
) do
|
||||
home = Streams::NodeRegistry.ensure_home_from_env!
|
||||
user = User.create!(email: "budget@example.com", name: "B", password: "Password123", role: "coach")
|
||||
club = Club.create!(name: "Budget", sport: "volleyball")
|
||||
team = club.teams.create!(name: "T", sport: "volleyball", slug: "budget-t")
|
||||
match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now)
|
||||
StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "live", stream_node: home)
|
||||
|
||||
provisioner = instance_double(Streams::NodeProvisioner)
|
||||
expect(provisioner).not_to receive(:provision_lab!)
|
||||
|
||||
result = described_class.reconcile!(provisioner: provisioner)
|
||||
expect(result.actions).to include(:blocked_capacity)
|
||||
expect(described_class.metrics[:within_budget]).to eq(true)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user