From cc9c4a11209df1b738f4abceb907c1c37c61644c Mon Sep 17 00:00:00 2001 From: Emiliano Frascaro Date: Wed, 16 Sep 2026 07:43:34 +0200 Subject: [PATCH] Chiude le sessioni idle abbandonate prima che occupino slot e scalino Hetzner. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Il sweeper ora termina le idle dopo 30 minuti (cron ogni 10), così non restano appese a tenere accesi i warm-spare. Co-authored-by: Cursor --- .../app/jobs/cleanup_expired_sessions_job.rb | 63 +++++++++++++++++-- backend/config/initializers/sidekiq_cron.rb | 5 +- backend/lib/tasks/streams_nodes.rake | 5 ++ .../jobs/cleanup_expired_sessions_job_spec.rb | 62 ++++++++++++++++++ infra/.env.collaudo.example | 2 + infra/.env.production.example | 3 + infra/scripts/install_production_cron.sh | 1 + 7 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 backend/spec/jobs/cleanup_expired_sessions_job_spec.rb diff --git a/backend/app/jobs/cleanup_expired_sessions_job.rb b/backend/app/jobs/cleanup_expired_sessions_job.rb index 91addb8..9935b11 100644 --- a/backend/app/jobs/cleanup_expired_sessions_job.rb +++ b/backend/app/jobs/cleanup_expired_sessions_job.rb @@ -1,12 +1,65 @@ +# frozen_string_literal: true + +# Chiude sessioni abbandonate che altrimenti occupano slot MediaMTX e +# fanno scattare warm-spare / scale-out Hetzner. +# +# - idle: wizard aperti e mai partiti (default 30 min) +# - connecting / reconnecting: hang senza go_live (default 6 h) class CleanupExpiredSessionsJob include Sidekiq::Job def perform - StreamSession.where(status: %w[connecting reconnecting]) - .where("updated_at < ?", 6.hours.ago) - .find_each do |session| - session.fail! if session.may_fail? - Mediamtx::Client.for_session(session).delete_path(session) + closed_idle = sweep_idle! + closed_stale = sweep_stale_connect! + Rails.logger.info( + "[CleanupExpiredSessionsJob] idle_closed=#{closed_idle} stale_closed=#{closed_stale} " \ + "idle_minutes=#{self.class.idle_minutes} stale_hours=#{self.class.stale_hours}" + ) + end + + def self.idle_minutes + ENV.fetch("STREAM_SESSION_IDLE_MINUTES", "30").to_i + end + + def self.stale_hours + ENV.fetch("STREAM_SESSION_STALE_HOURS", "6").to_i + end + + private + + def sweep_idle! + cutoff = self.class.idle_minutes.minutes.ago + count = 0 + StreamSession.where(status: "idle").where("updated_at < ?", cutoff).find_each do |session| + close!(session, reason: "idle_timeout") + count += 1 end + count + end + + def sweep_stale_connect! + cutoff = self.class.stale_hours.hours.ago + count = 0 + StreamSession.where(status: %w[connecting reconnecting]) + .where("updated_at < ?", cutoff) + .find_each do |session| + close!(session, reason: "stale_#{session.status}") + count += 1 + end + count + end + + def close!(session, reason:) + session.stream_events.create!( + event_type: "error", + metadata: { reason: reason }, + occurred_at: Time.current + ) + Sessions::Stop.new(session).call + rescue StandardError => e + Rails.logger.warn( + "[CleanupExpiredSessionsJob] session=#{session.id} reason=#{reason} " \ + "#{e.class}: #{e.message}" + ) end end diff --git a/backend/config/initializers/sidekiq_cron.rb b/backend/config/initializers/sidekiq_cron.rb index 57d20c9..71fe870 100644 --- a/backend/config/initializers/sidekiq_cron.rb +++ b/backend/config/initializers/sidekiq_cron.rb @@ -1,2 +1,3 @@ -# Schedule CleanupExpiredSessionsJob hourly via host cron or Kamal: -# 0 * * * * cd /app && bundle exec rails runner "CleanupExpiredSessionsJob.perform_async" +# CleanupExpiredSessionsJob: host cron via infra/scripts/install_production_cron.sh +# */10 * * * * ... streams:cleanup_expired_sessions +# Env: STREAM_SESSION_IDLE_MINUTES (default 30), STREAM_SESSION_STALE_HOURS (default 6) diff --git a/backend/lib/tasks/streams_nodes.rake b/backend/lib/tasks/streams_nodes.rake index 1ff4c5a..a40df4d 100644 --- a/backend/lib/tasks/streams_nodes.rake +++ b/backend/lib/tasks/streams_nodes.rake @@ -7,6 +7,11 @@ namespace :streams do puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}" end + desc "Chiude sessioni idle/stale che occupano slot (STREAM_SESSION_IDLE_MINUTES, default 30)" + task cleanup_expired_sessions: :environment do + CleanupExpiredSessionsJob.new.perform + end + namespace :nodes do desc "Assicura il nodo home dagli ENV MediaMTX" task ensure_home: :environment do diff --git a/backend/spec/jobs/cleanup_expired_sessions_job_spec.rb b/backend/spec/jobs/cleanup_expired_sessions_job_spec.rb new file mode 100644 index 0000000..6bee7eb --- /dev/null +++ b/backend/spec/jobs/cleanup_expired_sessions_job_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe CleanupExpiredSessionsJob do + let(:user) { User.create!(email: "idle-clean@test.it", name: "Idle", password: "Password123", role: "coach") } + let(:club) { Club.create!(name: "Club Idle", sport: "volleyball") } + let(:team) { club.teams.create!(name: "Team Idle", sport: "volleyball") } + let(:match) { team.matches.create!(opponent_name: "Opp", sport: "volleyball") } + + before do + club.club_memberships.create!(user: user, role: "owner") + allow(SessionChannel).to receive(:broadcast_message) + allow(Streams::YoutubeRelay).to receive(:stop) + client = instance_double(Mediamtx::Client, delete_path: true) + allow(Mediamtx::Client).to receive(:for_session).and_return(client) + allow(Recordings::FinalizeSession).to receive(:new).and_return( + instance_double(Recordings::FinalizeSession, call: nil) + ) + allow(Tournaments::CaptureStreamResult).to receive(:call) + end + + def create_session(status:, updated_at:) + StreamSession.create!( + match: match, + user: user, + platform: "youtube", + status: status, + privacy_status: "unlisted" + ).tap { |s| s.update_columns(updated_at: updated_at) } + end + + it "chiude sessioni idle oltre STREAM_SESSION_IDLE_MINUTES (default 30)" do + stale = create_session(status: "idle", updated_at: 45.minutes.ago) + fresh = create_session(status: "idle", updated_at: 10.minutes.ago) + + described_class.new.perform + + expect(stale.reload.status).to eq("ended") + expect(fresh.reload.status).to eq("idle") + expect(stale.stream_events.where(event_type: "error").exists?).to eq(true) + end + + it "chiude connecting/reconnecting stale oltre STREAM_SESSION_STALE_HOURS (default 6)" do + stale = create_session(status: "connecting", updated_at: 7.hours.ago) + fresh = create_session(status: "connecting", updated_at: 1.hour.ago) + + described_class.new.perform + + expect(stale.reload.status).to eq("ended") + expect(fresh.reload.status).to eq("connecting") + end + + it "non tocca sessioni live" do + live = create_session(status: "live", updated_at: 2.hours.ago) + live.update_columns(started_at: 2.hours.ago) + + described_class.new.perform + + expect(live.reload.status).to eq("live") + end +end diff --git a/infra/.env.collaudo.example b/infra/.env.collaudo.example index e860237..be78d60 100644 --- a/infra/.env.collaudo.example +++ b/infra/.env.collaudo.example @@ -112,6 +112,8 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0 STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2 STREAM_AUTOSCALE_WARM_SPARE=1 STREAM_AUTOSCALE_IDLE_MINUTES=30 +STREAM_SESSION_IDLE_MINUTES=30 +STREAM_SESSION_STALE_HOURS=6 STREAM_AUTOSCALE_MAX_NODES=12 STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015 STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150 diff --git a/infra/.env.production.example b/infra/.env.production.example index d8eafb8..81c91bc 100644 --- a/infra/.env.production.example +++ b/infra/.env.production.example @@ -148,6 +148,9 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0 STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2 STREAM_AUTOSCALE_WARM_SPARE=1 STREAM_AUTOSCALE_IDLE_MINUTES=30 +# Sessioni idle (wizard mai partito) / connecting-reconnecting stale: liberano slot MediaMTX +STREAM_SESSION_IDLE_MINUTES=30 +STREAM_SESSION_STALE_HOURS=6 # Fase A (qualità-first): soft ≈ 4 + 12×6 = 76. Fasi successive (dopo misure): # B: MAX_NODES=22 BUDGET=250 → ~136 # C: MAX_NODES=33 BUDGET=360 → ~202 diff --git a/infra/scripts/install_production_cron.sh b/infra/scripts/install_production_cron.sh index 23ad1c6..7bd45b7 100755 --- a/infra/scripts/install_production_cron.sh +++ b/infra/scripts/install_production_cron.sh @@ -33,6 +33,7 @@ CRON_BLOCK="${MARKER} 15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:aggregate >> ${LOG_DIR}/cron-analytics.log 2>&1 20 4 * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:purge >> ${LOG_DIR}/cron-analytics.log 2>&1 */15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:night_cloud_sweep >> ${LOG_DIR}/cron-night-sweep.log 2>&1 +*/10 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:cleanup_expired_sessions >> ${LOG_DIR}/cron-session-cleanup.log 2>&1 */10 * * * * mkdir -p ${LOG_DIR} && /bin/bash ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1 "