From 7c7b2bf14cce181496e8c43113181b1d30bb29ea Mon Sep 17 00:00:00 2001 From: Emiliano Frascaro Date: Sat, 5 Sep 2026 09:48:50 +0200 Subject: [PATCH] Alza soft capacity fase A e aggiunge quiet hours CPX notturne. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Home a 4 e cloud a 6 con MAX_NODES=12 (~76 soft); di notte (02–07 Europe/Rome) niente scale-out/warm-spare e sweeper che chiude i CPX idle. Co-authored-by: Cursor --- .cursor/rules/stream-soft-limit-scaling.mdc | 18 +-- .../jobs/streams/night_cloud_sweeper_job.rb | 33 +++++ backend/app/services/streams/autoscaler.rb | 13 +- .../services/streams/night_cloud_sweeper.rb | 83 +++++++++++++ backend/app/services/streams/quiet_hours.rb | 59 +++++++++ backend/config/initializers/sidekiq.rb | 1 + backend/lib/tasks/streams_nodes.rake | 6 + .../spec/services/streams/autoscaler_spec.rb | 35 ++++++ .../streams/night_cloud_sweeper_spec.rb | 114 ++++++++++++++++++ .../spec/services/streams/quiet_hours_spec.rb | 40 ++++++ docs/infrastructure/STREAMING_AUTOSCALE.md | 21 +++- infra/.env.collaudo.example | 12 +- infra/.env.production.example | 18 ++- infra/docker-compose.prod.yml | 24 ++-- infra/scripts/install_production_cron.sh | 1 + infra/scripts/stream_capacity_phase.sh | 60 +++++++++ 16 files changed, 513 insertions(+), 25 deletions(-) create mode 100644 backend/app/jobs/streams/night_cloud_sweeper_job.rb create mode 100644 backend/app/services/streams/night_cloud_sweeper.rb create mode 100644 backend/app/services/streams/quiet_hours.rb create mode 100644 backend/spec/services/streams/night_cloud_sweeper_spec.rb create mode 100644 backend/spec/services/streams/quiet_hours_spec.rb create mode 100755 infra/scripts/stream_capacity_phase.sh diff --git a/.cursor/rules/stream-soft-limit-scaling.mdc b/.cursor/rules/stream-soft-limit-scaling.mdc index 6160b57..9f9029c 100644 --- a/.cursor/rules/stream-soft-limit-scaling.mdc +++ b/.cursor/rules/stream-soft-limit-scaling.mdc @@ -9,14 +9,18 @@ Con la crescita del numero di clienti (e delle dirette concorrenti attese), **al Config rilevante (prod, tipicamente `infra/.env` + `StreamNode`): -| Parametro | Ruolo oggi (post load test 2026-08) | -|-----------|--------------------------------------| -| `STREAM_NODE_HOME_MAX_PUBLISHERS` | Soft home (es. 6) | -| `STREAM_CLOUD_MAX_PUBLISHERS` | Soft per CPX (es. 4; cpx12 ha tenuto 8 in probe) | -| `STREAM_AUTOSCALE_MAX_OVERFLOW` / max overflow nodes | Quanti CPX in parallelo (es. 3 → tetto cluster ≈ home + N×cloud) | +| Parametro | Ruolo (qualità-first 2026-09) | +|-----------|-------------------------------| +| `STREAM_NODE_HOME_MAX_PUBLISHERS` | Soft home (**4** — scale-out anticipato) | +| `STREAM_CLOUD_MAX_PUBLISHERS` | Soft per CPX (**6**; 8 solo dopo misure QoS) | +| `STREAM_AUTOSCALE_MAX_NODES` | Quanti CPX in parallelo (fase A **12** → B 22 → C 33) | | `STREAM_AUTOSCALE_SOFT_FREE_SLOTS` | Anticipo scale-out | +| `STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR` | Gate 24/7 (fase A **150**) | +| `STREAM_AUTOSCALE_QUIET_HOURS` | `02:00-07:00` Europe/Rome: no scale-out/warm-spare; sweeper chiude CPX idle | | Piano `concurrent_streams_limit` | Tetto **per club** (Premium Full = 10): indipendente dal cluster | -Capienza cluster soft ≈ `home_max + max_overflow × cloud_max` (es. 6+3×4 = **18**). +Capienza cluster soft ≈ `home_max + max_overflow × cloud_max` (fase A: 4+12×6 = **76**; fase C: 4+33×6 = **202**). -Quando si parla di capacity planning, deploy autoscale, o “troppe dirette”, ricordare di rivedere questi valori (e il limite piano) in base ai clienti reali — non lasciare i soft limit di collaudo/early-prod a lungo. +Quiet hours: `Streams::NightCloudSweeper` + blocco provision in `Streams::Autoscaler`. Nodi cloud con sessioni attive di notte → alert Ops, non spegnere. + +Quando si parla di capacity planning, deploy autoscale, o “troppe dirette”, ricordare di rivedere questi valori (e il limite piano) in base ai clienti reali — non lasciare i soft limit di collaudo/early-prod a lungo. Densità 8 solo dopo misure. diff --git a/backend/app/jobs/streams/night_cloud_sweeper_job.rb b/backend/app/jobs/streams/night_cloud_sweeper_job.rb new file mode 100644 index 0000000..a09d734 --- /dev/null +++ b/backend/app/jobs/streams/night_cloud_sweeper_job.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Streams + class NightCloudSweeperJob + include Sidekiq::Job + + sidekiq_options retry: 1, queue: "default" + + INTERVAL_SECS = ENV.fetch("STREAM_NIGHT_SWEEP_INTERVAL_SECS", "900").to_i + REDIS_CHAIN_KEY = "streams:night_sweep:chain" + + def self.ensure_chain + return unless redis + return if redis.get(REDIS_CHAIN_KEY) + + redis.set(REDIS_CHAIN_KEY, "1", ex: INTERVAL_SECS * 2) + perform_in(INTERVAL_SECS) + end + + def self.redis + @redis ||= Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) + rescue Redis::CannotConnectError + nil + end + + def perform + Streams::NightCloudSweeper.sweep! + ensure + self.class.redis&.set(REDIS_CHAIN_KEY, "1", ex: INTERVAL_SECS * 2) + self.class.perform_in(INTERVAL_SECS) + end + end +end diff --git a/backend/app/services/streams/autoscaler.rb b/backend/app/services/streams/autoscaler.rb index 95b5e60..7d113f3 100644 --- a/backend/app/services/streams/autoscaler.rb +++ b/backend/app/services/streams/autoscaler.rb @@ -71,6 +71,10 @@ module Streams estimated_monthly_eur(overflow_count) <= monthly_budget_eur end + def quiet_hours? + QuietHours.active? + end + def reconcile!(provisioner: nil) return Result.new(skipped: true, actions: [], metrics: metrics) unless enabled? @@ -106,7 +110,8 @@ module Streams allow_cloud: allow_cloud?, estimated_monthly_eur: estimated_monthly_eur(overflow.size), monthly_budget_eur: monthly_budget_eur, - within_budget: within_budget?(overflow.size) + within_budget: within_budget?(overflow.size), + quiet_hours: quiet_hours? } end @@ -201,6 +206,7 @@ module Streams end def warm_spare_desired?(m) + return false if self.class.quiet_hours? return false if self.class.warm_spare_min <= 0 need_capacity?(m) || overflow_in_use? @@ -211,6 +217,7 @@ module Streams end def can_provision?(m) + return false if self.class.quiet_hours? return false if m[:overflow_nodes] >= self.class.max_overflow_nodes return false unless self.class.within_budget?(m[:overflow_nodes] + 1) return false if self.class.kind == "cloud" && !self.class.allow_cloud? @@ -249,6 +256,9 @@ module Streams end def idle_long_enough?(node) + # Di notte chiudi subito i nodi idle (niente attesa IDLE_MINUTES). + return true if self.class.quiet_hours? + idle_since(node) <= self.class.idle_minutes.minutes.ago end @@ -258,6 +268,7 @@ module Streams end def keep_as_warm_spare?(node) + return false if self.class.quiet_hours? return false unless warm_spare_desired?(self.class.metrics) spares = StreamNode.ready.where.not(slug: NodeRegistry::HOME_SLUG).select { |n| n.active_publishers.zero? } diff --git a/backend/app/services/streams/night_cloud_sweeper.rb b/backend/app/services/streams/night_cloud_sweeper.rb new file mode 100644 index 0000000..84edbbe --- /dev/null +++ b/backend/app/services/streams/night_cloud_sweeper.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Streams + # Durante quiet hours chiude CPX cloud idle; se hanno sessioni attive solo alert Ops. + class NightCloudSweeper + Result = Struct.new(:skipped, :actions, :error, keyword_init: true) + + class << self + def enabled? + ENV.fetch("STREAM_NIGHT_SWEEP_ENABLED", "1") == "1" + end + + def sweep!(provisioner: nil) + new(provisioner: provisioner).sweep! + end + end + + def initialize(provisioner: nil) + @provisioner = provisioner || NodeProvisioner.new + end + + def sweep! + unless self.class.enabled? + return Result.new(skipped: true, actions: [], error: "disabled") + end + unless QuietHours.active? + return Result.new(skipped: true, actions: [], error: "outside_quiet_hours") + end + + actions = [] + cloud_nodes.find_each do |node| + if node.occupying_sessions.exists? + alert_active_night_node!(node) + actions << :"alert_active_#{node.slug}" + next + end + + close_idle_node!(node) + actions << :"decommission_#{node.slug}" + rescue NodeProvisioner::BusyError, NodeProvisioner::Error => e + Rails.logger.warn("[Streams::NightCloudSweeper] #{node.slug}: #{e.message}") + actions << :"error_#{node.slug}" + end + + Rails.logger.info("[Streams::NightCloudSweeper] actions=#{actions.inspect}") + Result.new(skipped: false, actions: actions) + end + + private + + def cloud_nodes + StreamNode.where(role: "cloud").where(status: %w[ready draining provisioning]) + end + + def alert_active_night_node!(node) + Ops::IncidentRecorder.record( + { + kind: "stream_overflow", + severity: "warning", + title: "CPX attivo di notte — verificare evento", + message: "Nodo #{node.slug} ha #{node.active_publishers} sessione/i in quiet hours (#{QuietHours.range_config} #{QuietHours.time_zone_name})", + metadata: { + "slug" => node.slug, + "session_ids" => node.occupying_sessions.pluck(:id), + "quiet_hours" => QuietHours.range_config + }, + fingerprint: "stream_overflow:night_active:#{node.slug}" + } + ) + end + + def close_idle_node!(node) + @provisioner.drain!(node) unless node.status == "draining" + node.reload + raise NodeProvisioner::BusyError, "sessioni ancora attive" if node.occupying_sessions.exists? + + slug = node.slug + @provisioner.decommission!(node) + Rails.logger.warn("[Streams::NightCloudSweeper] decommissioned idle cloud node #{slug} during quiet hours") + Ops::IncidentRecorder.resolve(fingerprint: "stream_overflow:night_active:#{slug}") + end + end +end diff --git a/backend/app/services/streams/quiet_hours.rb b/backend/app/services/streams/quiet_hours.rb new file mode 100644 index 0000000..7bb69cf --- /dev/null +++ b/backend/app/services/streams/quiet_hours.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Streams + # Finestra notturna (default 02:00–07:00 Europe/Rome) in cui non devono restare CPX idle. + module QuietHours + DEFAULT_RANGE = "02:00-07:00" + DEFAULT_TZ = "Europe/Rome" + + module_function + + def range_config + ENV.fetch("STREAM_AUTOSCALE_QUIET_HOURS", DEFAULT_RANGE).to_s.strip + end + + def time_zone_name + ENV.fetch("STREAM_AUTOSCALE_QUIET_TZ", DEFAULT_TZ) + end + + def configured? + range_config.present? && range_config != "off" && range_config != "0" + end + + def active?(now: Time.current) + return false unless configured? + + zone = ActiveSupport::TimeZone[time_zone_name] || Time.find_zone!(time_zone_name) + local = now.in_time_zone(zone) + start_min, end_min = parse_range(range_config) + return false if start_min.nil? || end_min.nil? + + current = local.hour * 60 + local.min + if start_min <= end_min + current >= start_min && current < end_min + else + # es. 22:00-06:00 + current >= start_min || current < end_min + end + rescue ArgumentError => e + Rails.logger.warn("[Streams::QuietHours] invalid config: #{e.message}") + false + end + + def parse_range(raw) + start_s, end_s = raw.split("-", 2).map { |p| p.to_s.strip } + return [nil, nil] if start_s.blank? || end_s.blank? + + [parse_hhmm(start_s), parse_hhmm(end_s)] + end + + def parse_hhmm(value) + h, m = value.split(":", 2) + hours = Integer(h) + mins = Integer(m || 0) + raise ArgumentError, "ora fuori range: #{value}" unless hours.between?(0, 23) && mins.between?(0, 59) + + hours * 60 + mins + end + end +end diff --git a/backend/config/initializers/sidekiq.rb b/backend/config/initializers/sidekiq.rb index 307d522..5d16355 100644 --- a/backend/config/initializers/sidekiq.rb +++ b/backend/config/initializers/sidekiq.rb @@ -7,6 +7,7 @@ Sidekiq.configure_server do |config| StreamPublisherSyncJob.ensure_chain Ops::HealthMonitorJob.ensure_chain Streams::AutoscalerJob.ensure_chain + Streams::NightCloudSweeperJob.ensure_chain end end diff --git a/backend/lib/tasks/streams_nodes.rake b/backend/lib/tasks/streams_nodes.rake index 8e7f446..1ff4c5a 100644 --- a/backend/lib/tasks/streams_nodes.rake +++ b/backend/lib/tasks/streams_nodes.rake @@ -1,6 +1,12 @@ # frozen_string_literal: true namespace :streams do + desc "Chiude CPX cloud idle in quiet hours (02:00–07:00 Europe/Rome); alert se sessioni attive" + task night_cloud_sweep: :environment do + result = Streams::NightCloudSweeper.sweep! + puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}" + end + namespace :nodes do desc "Assicura il nodo home dagli ENV MediaMTX" task ensure_home: :environment do diff --git a/backend/spec/services/streams/autoscaler_spec.rb b/backend/spec/services/streams/autoscaler_spec.rb index f86665e..c00dda9 100644 --- a/backend/spec/services/streams/autoscaler_spec.rb +++ b/backend/spec/services/streams/autoscaler_spec.rb @@ -19,6 +19,11 @@ RSpec.describe Streams::Autoscaler do redis.del(Streams::DnsProviders::Lab::REDIS_KEY) StreamSession.where.not(stream_node_id: nil).update_all(stream_node_id: nil, status: "ended", ended_at: Time.current) StreamNode.where.not(slug: Streams::NodeRegistry::HOME_SLUG).delete_all + ENV["STREAM_AUTOSCALE_QUIET_HOURS"] = "off" + end + + after do + ENV.delete("STREAM_AUTOSCALE_QUIET_HOURS") end it "is a no-op when disabled" do @@ -261,4 +266,34 @@ RSpec.describe Streams::Autoscaler do expect(node.reload.status).to eq("ready") end end + + it "blocks scale-out and warm spare during quiet hours" do + with_env( + "STREAM_AUTOSCALE_ENABLED" => "1", + "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00", + "STREAM_AUTOSCALE_SOFT_FREE_SLOTS" => "2", + "STREAM_AUTOSCALE_WARM_SPARE" => "1", + "STREAM_AUTOSCALE_KIND" => "lab", + "STREAM_AUTOSCALE_MAX_NODES" => "5", + "STREAM_NODE_HOME_MAX_PUBLISHERS" => "1", + "MEDIAMTX_API_URL" => "http://mtx-home:9997", + "MEDIAMTX_RTMP_URL" => "rtmp://home.example:1935", + "HLS_PUBLIC_URL" => "https://home.example/hls" + ) do + allow(Streams::QuietHours).to receive(:active?).and_return(true) + home = Streams::NodeRegistry.ensure_home_from_env! + user = User.create!(email: "qh@example.com", name: "Q", password: "Password123", role: "coach") + club = Club.create!(name: "QH", sport: "volleyball") + team = club.teams.create!(name: "T", sport: "volleyball", slug: "qh-t") + match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now) + StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "live", stream_node: home) + + provisioner = instance_double(Streams::NodeProvisioner) + expect(provisioner).not_to receive(:provision_lab!) + + result = described_class.reconcile!(provisioner: provisioner) + expect(result.actions).to include(:blocked_capacity) + expect(result.metrics[:quiet_hours]).to eq(true) + end + end end diff --git a/backend/spec/services/streams/night_cloud_sweeper_spec.rb b/backend/spec/services/streams/night_cloud_sweeper_spec.rb new file mode 100644 index 0000000..0027f24 --- /dev/null +++ b/backend/spec/services/streams/night_cloud_sweeper_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Streams::NightCloudSweeper do + def with_env(vars) + previous = vars.keys.index_with { |k| ENV[k] } + vars.each { |k, v| ENV[k] = v } + yield + ensure + previous.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v } + end + + def cloud_node!(slug: "ingest-cloud-01") + StreamNode.create!( + slug: slug, + hostname: "#{slug}.mltv-stream.net", + role: "cloud", + status: "ready", + provider: "hetzner", + provider_instance_id: "42", + rtmp_base_url: "rtmp://#{slug}:1935", + hls_base_url: "https://#{slug}/hls", + api_base_url: "http://10.0.0.9:9997", + max_publishers: 6, + max_relays: 6, + metadata: { "public_ip" => "1.2.3.4" } + ) + end + + before do + StreamSession.where.not(stream_node_id: nil).update_all(stream_node_id: nil, status: "ended", ended_at: Time.current) + StreamNode.where(role: "cloud").delete_all + end + + it "skips outside quiet hours" do + with_env( + "STREAM_NIGHT_SWEEP_ENABLED" => "1", + "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00", + "STREAM_AUTOSCALE_QUIET_TZ" => "Europe/Rome" + ) do + allow(Streams::QuietHours).to receive(:active?).and_return(false) + cloud_node! + result = described_class.sweep! + expect(result.skipped).to eq(true) + expect(StreamNode.where(role: "cloud").count).to eq(1) + end + end + + it "decommissions idle cloud nodes during quiet hours" do + with_env("STREAM_NIGHT_SWEEP_ENABLED" => "1") do + allow(Streams::QuietHours).to receive(:active?).and_return(true) + node = cloud_node! + provisioner = instance_double(Streams::NodeProvisioner) + allow(provisioner).to receive(:drain!) do |n| + n.update!(status: "draining") + n + end + allow(provisioner).to receive(:decommission!) do |n| + n.destroy! + true + end + + result = described_class.sweep!(provisioner: provisioner) + expect(result.skipped).to eq(false) + expect(result.actions).to include(:"decommission_#{node.slug}") + expect(StreamNode.find_by(id: node.id)).to be_nil + end + end + + it "alerts and keeps nodes with active sessions" do + with_env("STREAM_NIGHT_SWEEP_ENABLED" => "1") do + allow(Streams::QuietHours).to receive(:active?).and_return(true) + node = cloud_node! + user = User.create!(email: "night@example.com", name: "N", password: "Password123", role: "coach") + club = Club.create!(name: "Night", sport: "volleyball") + team = club.teams.create!(name: "T", sport: "volleyball", slug: "night-t") + match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now) + StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "live", stream_node: node) + + allow(Ops::IncidentRecorder).to receive(:record) + provisioner = instance_double(Streams::NodeProvisioner) + expect(provisioner).not_to receive(:decommission!) + + result = described_class.sweep!(provisioner: provisioner) + expect(result.actions).to include(:"alert_active_#{node.slug}") + expect(Ops::IncidentRecorder).to have_received(:record).with(hash_including( + fingerprint: "stream_overflow:night_active:#{node.slug}" + )) + expect(node.reload.status).to eq("ready") + end + end + + it "does not touch lab nodes" do + with_env("STREAM_NIGHT_SWEEP_ENABLED" => "1") do + allow(Streams::QuietHours).to receive(:active?).and_return(true) + StreamNode.create!( + slug: "ingest-lab-night-01", + hostname: "lab-night.local", + role: "lab", + status: "ready", + provider: "local", + rtmp_base_url: "rtmp://lab:1935", + hls_base_url: "https://lab/hls", + api_base_url: "http://lab:9997", + max_publishers: 2, + max_relays: 2 + ) + result = described_class.sweep! + expect(result.actions).to eq([]) + expect(StreamNode.find_by(slug: "ingest-lab-night-01")).to be_present + end + end +end diff --git a/backend/spec/services/streams/quiet_hours_spec.rb b/backend/spec/services/streams/quiet_hours_spec.rb new file mode 100644 index 0000000..c1c7b24 --- /dev/null +++ b/backend/spec/services/streams/quiet_hours_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Streams::QuietHours do + def with_env(vars) + previous = vars.keys.index_with { |k| ENV[k] } + vars.each { |k, v| ENV[k] = v } + yield + ensure + previous.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v } + end + + it "is active inside the Rome window" do + with_env( + "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00", + "STREAM_AUTOSCALE_QUIET_TZ" => "Europe/Rome" + ) do + now = Time.find_zone!("Europe/Rome").local(2026, 9, 5, 3, 30) + expect(described_class.active?(now: now)).to eq(true) + end + end + + it "is inactive outside the window" do + with_env( + "STREAM_AUTOSCALE_QUIET_HOURS" => "02:00-07:00", + "STREAM_AUTOSCALE_QUIET_TZ" => "Europe/Rome" + ) do + now = Time.find_zone!("Europe/Rome").local(2026, 9, 5, 10, 0) + expect(described_class.active?(now: now)).to eq(false) + end + end + + it "can be disabled with off" do + with_env("STREAM_AUTOSCALE_QUIET_HOURS" => "off") do + now = Time.find_zone!("Europe/Rome").local(2026, 9, 5, 3, 0) + expect(described_class.active?(now: now)).to eq(false) + end + end +end diff --git a/docs/infrastructure/STREAMING_AUTOSCALE.md b/docs/infrastructure/STREAMING_AUTOSCALE.md index d44791f..6f15c7f 100644 --- a/docs/infrastructure/STREAMING_AUTOSCALE.md +++ b/docs/infrastructure/STREAMING_AUTOSCALE.md @@ -376,10 +376,29 @@ HCLOUD_USER_DATA_FILE=/opt/matchlivetv/infra/stream-node/cloud-init.yaml STREAM_DNS_ZONE=mltv-stream.net STREAM_DNS_TTL=60 STREAM_CLOUD_DNS_SUFFIX=mltv-stream.net -STREAM_CLOUD_MAX_PUBLISHERS=4 +STREAM_CLOUD_MAX_PUBLISHERS=6 +STREAM_NODE_HOME_MAX_PUBLISHERS=4 STREAM_NODE_ENV=prod +STREAM_AUTOSCALE_MAX_NODES=12 +STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150 +STREAM_AUTOSCALE_QUIET_HOURS=02:00-07:00 +STREAM_AUTOSCALE_QUIET_TZ=Europe/Rome +STREAM_NIGHT_SWEEP_ENABLED=1 + # Default lab sicuro; in prod per overflow usa hetzner esplicitamente dal bottone admin STREAM_CLOUD_PROVIDER=local_lab STREAM_DNS_PROVIDER=lab ``` + +### Quiet hours + capacity phases (qualità-first) + +| Fase | home | cloud/CPX | MAX_NODES | budget soft | soft totale | +|------|------|-----------|-----------|-------------|-------------| +| A (ora) | 4 | 6 | 12 | 150 € | ~76 | +| B (post-misure) | 4 | 6 | 22 | 250 € | ~136 | +| C (tetto ≥200) | 4 | 6 | 33 | 360 € | ~202 | + +Densità **8**/CPX solo dopo misure QoS. Di notte (`Streams::QuietHours` + `Streams::NightCloudSweeper`): niente scale-out/warm-spare; CPX idle → decommission; CPX con sessioni → alert Ops. + +**Misure prima di fase B/C o densità 8:** CPU/RAM/ffmpeg su CPX a 4–6 publisher; slate/pause/HLS; relay YouTube (`RELAY_MAX_CONCURRENT`); tempo provision→ready; fattura Hetzner vs stima budget 24/7. \ No newline at end of file diff --git a/infra/.env.collaudo.example b/infra/.env.collaudo.example index f3e19ff..e860237 100644 --- a/infra/.env.collaudo.example +++ b/infra/.env.collaudo.example @@ -98,10 +98,11 @@ HCLOUD_USER_DATA_FILE=/opt/matchlivetv/infra/stream-node/cloud-init.yaml STREAM_DNS_ZONE=mltv-stream.net STREAM_DNS_TTL=60 STREAM_CLOUD_DNS_SUFFIX=mltv-stream.net -STREAM_CLOUD_MAX_PUBLISHERS=4 +STREAM_CLOUD_MAX_PUBLISHERS=6 +STREAM_NODE_HOME_MAX_PUBLISHERS=4 STREAM_NODE_ENV=collaudo STREAM_CLOUD_PROVIDER=local_lab -RELAY_MAX_CONCURRENT=4 +RELAY_MAX_CONCURRENT=6 YOUTUBE_RELAY_WORKER=1 # Worker home: STREAM_NODE_SLUG=home (default). Overflow: coda youtube_relay_ @@ -111,7 +112,10 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0 STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2 STREAM_AUTOSCALE_WARM_SPARE=1 STREAM_AUTOSCALE_IDLE_MINUTES=30 -STREAM_AUTOSCALE_MAX_NODES=5 +STREAM_AUTOSCALE_MAX_NODES=12 STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015 -STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=40 +STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150 +STREAM_AUTOSCALE_QUIET_HOURS=02:00-07:00 +STREAM_AUTOSCALE_QUIET_TZ=Europe/Rome +STREAM_NIGHT_SWEEP_ENABLED=1 STREAM_OVERFLOW_ORPHAN_HOURS=3 diff --git a/infra/.env.production.example b/infra/.env.production.example index 354543a..d8eafb8 100644 --- a/infra/.env.production.example +++ b/infra/.env.production.example @@ -131,11 +131,12 @@ HCLOUD_USER_DATA_FILE=/opt/matchlivetv/infra/stream-node/cloud-init.yaml STREAM_DNS_ZONE=mltv-stream.net STREAM_DNS_TTL=60 STREAM_CLOUD_DNS_SUFFIX=mltv-stream.net -STREAM_CLOUD_MAX_PUBLISHERS=4 +STREAM_CLOUD_MAX_PUBLISHERS=6 +STREAM_NODE_HOME_MAX_PUBLISHERS=4 STREAM_NODE_ENV=prod # Lab locale di default; il bottone admin "Provisiona nodo Hetzner" usa comunque i provider hetzner. STREAM_CLOUD_PROVIDER=local_lab -RELAY_MAX_CONCURRENT=4 +RELAY_MAX_CONCURRENT=6 YOUTUBE_RELAY_WORKER=1 # Deve coincidere con cloud-init (default mediamtx_webhook_dev_secret) — home chiama l'agent :9100. STREAM_NODE_AGENT_SECRET=mediamtx_webhook_dev_secret @@ -147,8 +148,17 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0 STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2 STREAM_AUTOSCALE_WARM_SPARE=1 STREAM_AUTOSCALE_IDLE_MINUTES=30 -STREAM_AUTOSCALE_MAX_NODES=5 +# Fase A (qualità-first): soft ≈ 4 + 12×6 = 76. Fasi successive (dopo misure): +# B: MAX_NODES=22 BUDGET=250 → ~136 +# C: MAX_NODES=33 BUDGET=360 → ~202 +# Densità 8 solo dopo misure QoS — non alzare CLOUD_MAX prima. +STREAM_AUTOSCALE_MAX_NODES=12 STREAM_AUTOSCALE_INTERVAL_SECS=60 STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015 -STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=40 +STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150 +# Quiet hours: zero CPX idle 02:00–07:00 Europe/Rome (sweeper + no scale-out/warm-spare) +STREAM_AUTOSCALE_QUIET_HOURS=02:00-07:00 +STREAM_AUTOSCALE_QUIET_TZ=Europe/Rome +STREAM_NIGHT_SWEEP_ENABLED=1 +STREAM_NIGHT_SWEEP_INTERVAL_SECS=900 STREAM_OVERFLOW_ORPHAN_HOURS=3 diff --git a/infra/docker-compose.prod.yml b/infra/docker-compose.prod.yml index 687fae7..5d1b2b1 100644 --- a/infra/docker-compose.prod.yml +++ b/infra/docker-compose.prod.yml @@ -73,7 +73,7 @@ services: YOUTUBE_REDIRECT_URI: ${YOUTUBE_REDIRECT_URI:-} YOUTUBE_PLATFORM_REFRESH_TOKEN: ${YOUTUBE_PLATFORM_REFRESH_TOKEN:-} STREAM_NODE_AGENT_SECRET: ${STREAM_NODE_AGENT_SECRET:-mediamtx_webhook_dev_secret} - STREAM_NODE_HOME_MAX_PUBLISHERS: ${STREAM_NODE_HOME_MAX_PUBLISHERS:-6} + STREAM_NODE_HOME_MAX_PUBLISHERS: ${STREAM_NODE_HOME_MAX_PUBLISHERS:-4} STREAM_CLOUD_PROVIDER: ${STREAM_CLOUD_PROVIDER:-local_lab} STREAM_DNS_PROVIDER: ${STREAM_DNS_PROVIDER:-hetzner} STREAM_AUTOSCALE_ENABLED: ${STREAM_AUTOSCALE_ENABLED:-0} @@ -81,17 +81,21 @@ services: STREAM_AUTOSCALE_ALLOW_CLOUD: ${STREAM_AUTOSCALE_ALLOW_CLOUD:-0} STREAM_AUTOSCALE_SOFT_FREE_SLOTS: ${STREAM_AUTOSCALE_SOFT_FREE_SLOTS:-2} STREAM_AUTOSCALE_WARM_SPARE: ${STREAM_AUTOSCALE_WARM_SPARE:-1} - STREAM_AUTOSCALE_MAX_NODES: ${STREAM_AUTOSCALE_MAX_NODES:-3} + STREAM_AUTOSCALE_MAX_NODES: ${STREAM_AUTOSCALE_MAX_NODES:-12} STREAM_AUTOSCALE_IDLE_MINUTES: ${STREAM_AUTOSCALE_IDLE_MINUTES:-30} STREAM_AUTOSCALE_INTERVAL_SECS: ${STREAM_AUTOSCALE_INTERVAL_SECS:-60} STREAM_AUTOSCALE_NODE_EUR_PER_HOUR: ${STREAM_AUTOSCALE_NODE_EUR_PER_HOUR:-0.015} - STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR: ${STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR:-40} + STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR: ${STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR:-150} + STREAM_AUTOSCALE_QUIET_HOURS: ${STREAM_AUTOSCALE_QUIET_HOURS:-02:00-07:00} + STREAM_AUTOSCALE_QUIET_TZ: ${STREAM_AUTOSCALE_QUIET_TZ:-Europe/Rome} + STREAM_NIGHT_SWEEP_ENABLED: ${STREAM_NIGHT_SWEEP_ENABLED:-1} + STREAM_NIGHT_SWEEP_INTERVAL_SECS: ${STREAM_NIGHT_SWEEP_INTERVAL_SECS:-900} STREAM_CLOUD_PUBLIC_CONTROL: ${STREAM_CLOUD_PUBLIC_CONTROL:-1} STREAM_NODE_ENV: ${STREAM_NODE_ENV:-prod} STREAM_DNS_ZONE: ${STREAM_DNS_ZONE:-mltv-stream.net} STREAM_DNS_TTL: ${STREAM_DNS_TTL:-60} STREAM_CLOUD_DNS_SUFFIX: ${STREAM_CLOUD_DNS_SUFFIX:-mltv-stream.net} - STREAM_CLOUD_MAX_PUBLISHERS: ${STREAM_CLOUD_MAX_PUBLISHERS:-4} + STREAM_CLOUD_MAX_PUBLISHERS: ${STREAM_CLOUD_MAX_PUBLISHERS:-6} HCLOUD_TOKEN: ${HCLOUD_TOKEN:-} HCLOUD_LOCATION: ${HCLOUD_LOCATION:-nbg1} HCLOUD_SERVER_TYPE: ${HCLOUD_SERVER_TYPE:-cpx12} @@ -219,7 +223,7 @@ services: YOUTUBE_PLATFORM_REFRESH_TOKEN: ${YOUTUBE_PLATFORM_REFRESH_TOKEN:-} YOUTUBE_RELAY_WORKER: "1" STREAM_NODE_SLUG: home - STREAM_NODE_HOME_MAX_PUBLISHERS: ${STREAM_NODE_HOME_MAX_PUBLISHERS:-6} + STREAM_NODE_HOME_MAX_PUBLISHERS: ${STREAM_NODE_HOME_MAX_PUBLISHERS:-4} STREAM_CLOUD_PROVIDER: ${STREAM_CLOUD_PROVIDER:-local_lab} STREAM_DNS_PROVIDER: ${STREAM_DNS_PROVIDER:-hetzner} STREAM_AUTOSCALE_ENABLED: ${STREAM_AUTOSCALE_ENABLED:-0} @@ -227,17 +231,21 @@ services: STREAM_AUTOSCALE_ALLOW_CLOUD: ${STREAM_AUTOSCALE_ALLOW_CLOUD:-0} STREAM_AUTOSCALE_SOFT_FREE_SLOTS: ${STREAM_AUTOSCALE_SOFT_FREE_SLOTS:-2} STREAM_AUTOSCALE_WARM_SPARE: ${STREAM_AUTOSCALE_WARM_SPARE:-1} - STREAM_AUTOSCALE_MAX_NODES: ${STREAM_AUTOSCALE_MAX_NODES:-3} + STREAM_AUTOSCALE_MAX_NODES: ${STREAM_AUTOSCALE_MAX_NODES:-12} STREAM_AUTOSCALE_IDLE_MINUTES: ${STREAM_AUTOSCALE_IDLE_MINUTES:-30} STREAM_AUTOSCALE_INTERVAL_SECS: ${STREAM_AUTOSCALE_INTERVAL_SECS:-60} STREAM_AUTOSCALE_NODE_EUR_PER_HOUR: ${STREAM_AUTOSCALE_NODE_EUR_PER_HOUR:-0.015} - STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR: ${STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR:-40} + STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR: ${STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR:-150} + STREAM_AUTOSCALE_QUIET_HOURS: ${STREAM_AUTOSCALE_QUIET_HOURS:-02:00-07:00} + STREAM_AUTOSCALE_QUIET_TZ: ${STREAM_AUTOSCALE_QUIET_TZ:-Europe/Rome} + STREAM_NIGHT_SWEEP_ENABLED: ${STREAM_NIGHT_SWEEP_ENABLED:-1} + STREAM_NIGHT_SWEEP_INTERVAL_SECS: ${STREAM_NIGHT_SWEEP_INTERVAL_SECS:-900} STREAM_CLOUD_PUBLIC_CONTROL: ${STREAM_CLOUD_PUBLIC_CONTROL:-1} STREAM_NODE_ENV: ${STREAM_NODE_ENV:-prod} STREAM_DNS_ZONE: ${STREAM_DNS_ZONE:-mltv-stream.net} STREAM_DNS_TTL: ${STREAM_DNS_TTL:-60} STREAM_CLOUD_DNS_SUFFIX: ${STREAM_CLOUD_DNS_SUFFIX:-mltv-stream.net} - STREAM_CLOUD_MAX_PUBLISHERS: ${STREAM_CLOUD_MAX_PUBLISHERS:-4} + STREAM_CLOUD_MAX_PUBLISHERS: ${STREAM_CLOUD_MAX_PUBLISHERS:-6} HCLOUD_TOKEN: ${HCLOUD_TOKEN:-} HCLOUD_LOCATION: ${HCLOUD_LOCATION:-nbg1} HCLOUD_SERVER_TYPE: ${HCLOUD_SERVER_TYPE:-cpx12} diff --git a/infra/scripts/install_production_cron.sh b/infra/scripts/install_production_cron.sh index 7df517d..23ad1c6 100755 --- a/infra/scripts/install_production_cron.sh +++ b/infra/scripts/install_production_cron.sh @@ -32,6 +32,7 @@ CRON_BLOCK="${MARKER} 0 8 * * * mkdir -p ${LOG_DIR} && ${RUNNER} recordings:expiry_warnings >> ${LOG_FILE} 2>&1 15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:aggregate >> ${LOG_DIR}/cron-analytics.log 2>&1 20 4 * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:purge >> ${LOG_DIR}/cron-analytics.log 2>&1 +*/15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:night_cloud_sweep >> ${LOG_DIR}/cron-night-sweep.log 2>&1 */10 * * * * mkdir -p ${LOG_DIR} && /bin/bash ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1 " diff --git a/infra/scripts/stream_capacity_phase.sh b/infra/scripts/stream_capacity_phase.sh new file mode 100755 index 0000000..262911c --- /dev/null +++ b/infra/scripts/stream_capacity_phase.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Applica soft-limit capacity phases su infra/.env (qualità-first). +# Uso (sul server, in /opt/matchlivetv/infra): +# bash ../scripts/deploy/../infra/scripts/stream_capacity_phase.sh A|B|C +# Fase B/C solo DOPO misure QoS/costi — non saltare. +set -euo pipefail + +PHASE="${1:-}" +INFRA="$(cd "$(dirname "$0")/.." && pwd)" +ENV_FILE="${INFRA}/.env" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Manca $ENV_FILE" >&2 + exit 1 +fi + +case "$PHASE" in + A) + HOME_MAX=4; CLOUD_MAX=6; MAX_NODES=12; BUDGET=150; SOFT_FREE=2; WARM=1; RELAY=6 + ;; + B) + HOME_MAX=4; CLOUD_MAX=6; MAX_NODES=22; BUDGET=250; SOFT_FREE=3; WARM=2; RELAY=6 + ;; + C) + HOME_MAX=4; CLOUD_MAX=6; MAX_NODES=33; BUDGET=360; SOFT_FREE=4; WARM=2; RELAY=6 + ;; + *) + echo "Uso: $0 A|B|C" >&2 + echo " A → soft ~76 (home4 + 12×6)" >&2 + echo " B → soft ~136 (dopo misure)" >&2 + echo " C → soft ~202 (tetto ≥200)" >&2 + exit 1 + ;; +esac + +cp -a "$ENV_FILE" "${ENV_FILE}.bak.phase${PHASE}-$(date +%Y%m%d%H%M%S)" + +set_kv() { + local key="$1" val="$2" + if grep -qE "^${key}=" "$ENV_FILE"; then + sed -i "s|^${key}=.*|${key}=${val}|" "$ENV_FILE" + else + printf '%s=%s\n' "$key" "$val" >> "$ENV_FILE" + fi +} + +set_kv STREAM_NODE_HOME_MAX_PUBLISHERS "$HOME_MAX" +set_kv STREAM_CLOUD_MAX_PUBLISHERS "$CLOUD_MAX" +set_kv STREAM_AUTOSCALE_MAX_NODES "$MAX_NODES" +set_kv STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR "$BUDGET" +set_kv STREAM_AUTOSCALE_SOFT_FREE_SLOTS "$SOFT_FREE" +set_kv STREAM_AUTOSCALE_WARM_SPARE "$WARM" +set_kv RELAY_MAX_CONCURRENT "$RELAY" +set_kv STREAM_AUTOSCALE_QUIET_HOURS "02:00-07:00" +set_kv STREAM_AUTOSCALE_QUIET_TZ "Europe/Rome" +set_kv STREAM_NIGHT_SWEEP_ENABLED "1" + +echo "Fase $PHASE applicata su $ENV_FILE" +echo " home=$HOME_MAX cloud=$CLOUD_MAX max_nodes=$MAX_NODES budget=$BUDGET soft≈$((HOME_MAX + MAX_NODES * CLOUD_MAX))" +echo "Riavvia rails+sidekiq e: rails runner 'Streams::NodeRegistry.ensure_home_from_env!'"