Chiude le sessioni idle abbandonate prima che occupino slot e scalino Hetzner.

Il sweeper ora termina le idle dopo 30 minuti (cron ogni 10), così non restano appese a tenere accesi i warm-spare.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-09-16 07:43:34 +02:00
co-authored by Cursor
parent 2db6541c91
commit cc9c4a1120
7 changed files with 134 additions and 7 deletions
@@ -1,12 +1,65 @@
# frozen_string_literal: true
# Chiude sessioni abbandonate che altrimenti occupano slot MediaMTX e
# fanno scattare warm-spare / scale-out Hetzner.
#
# - idle: wizard aperti e mai partiti (default 30 min)
# - connecting / reconnecting: hang senza go_live (default 6 h)
class CleanupExpiredSessionsJob class CleanupExpiredSessionsJob
include Sidekiq::Job include Sidekiq::Job
def perform def perform
StreamSession.where(status: %w[connecting reconnecting]) closed_idle = sweep_idle!
.where("updated_at < ?", 6.hours.ago) closed_stale = sweep_stale_connect!
.find_each do |session| Rails.logger.info(
session.fail! if session.may_fail? "[CleanupExpiredSessionsJob] idle_closed=#{closed_idle} stale_closed=#{closed_stale} " \
Mediamtx::Client.for_session(session).delete_path(session) "idle_minutes=#{self.class.idle_minutes} stale_hours=#{self.class.stale_hours}"
)
end
def self.idle_minutes
ENV.fetch("STREAM_SESSION_IDLE_MINUTES", "30").to_i
end
def self.stale_hours
ENV.fetch("STREAM_SESSION_STALE_HOURS", "6").to_i
end
private
def sweep_idle!
cutoff = self.class.idle_minutes.minutes.ago
count = 0
StreamSession.where(status: "idle").where("updated_at < ?", cutoff).find_each do |session|
close!(session, reason: "idle_timeout")
count += 1
end end
count
end
def sweep_stale_connect!
cutoff = self.class.stale_hours.hours.ago
count = 0
StreamSession.where(status: %w[connecting reconnecting])
.where("updated_at < ?", cutoff)
.find_each do |session|
close!(session, reason: "stale_#{session.status}")
count += 1
end
count
end
def close!(session, reason:)
session.stream_events.create!(
event_type: "error",
metadata: { reason: reason },
occurred_at: Time.current
)
Sessions::Stop.new(session).call
rescue StandardError => e
Rails.logger.warn(
"[CleanupExpiredSessionsJob] session=#{session.id} reason=#{reason} " \
"#{e.class}: #{e.message}"
)
end end
end end
+3 -2
View File
@@ -1,2 +1,3 @@
# Schedule CleanupExpiredSessionsJob hourly via host cron or Kamal: # CleanupExpiredSessionsJob: host cron via infra/scripts/install_production_cron.sh
# 0 * * * * cd /app && bundle exec rails runner "CleanupExpiredSessionsJob.perform_async" # */10 * * * * ... streams:cleanup_expired_sessions
# Env: STREAM_SESSION_IDLE_MINUTES (default 30), STREAM_SESSION_STALE_HOURS (default 6)
+5
View File
@@ -7,6 +7,11 @@ namespace :streams do
puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}" puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}"
end end
desc "Chiude sessioni idle/stale che occupano slot (STREAM_SESSION_IDLE_MINUTES, default 30)"
task cleanup_expired_sessions: :environment do
CleanupExpiredSessionsJob.new.perform
end
namespace :nodes do namespace :nodes do
desc "Assicura il nodo home dagli ENV MediaMTX" desc "Assicura il nodo home dagli ENV MediaMTX"
task ensure_home: :environment do task ensure_home: :environment do
@@ -0,0 +1,62 @@
# frozen_string_literal: true
require "rails_helper"
RSpec.describe CleanupExpiredSessionsJob do
let(:user) { User.create!(email: "idle-clean@test.it", name: "Idle", password: "Password123", role: "coach") }
let(:club) { Club.create!(name: "Club Idle", sport: "volleyball") }
let(:team) { club.teams.create!(name: "Team Idle", sport: "volleyball") }
let(:match) { team.matches.create!(opponent_name: "Opp", sport: "volleyball") }
before do
club.club_memberships.create!(user: user, role: "owner")
allow(SessionChannel).to receive(:broadcast_message)
allow(Streams::YoutubeRelay).to receive(:stop)
client = instance_double(Mediamtx::Client, delete_path: true)
allow(Mediamtx::Client).to receive(:for_session).and_return(client)
allow(Recordings::FinalizeSession).to receive(:new).and_return(
instance_double(Recordings::FinalizeSession, call: nil)
)
allow(Tournaments::CaptureStreamResult).to receive(:call)
end
def create_session(status:, updated_at:)
StreamSession.create!(
match: match,
user: user,
platform: "youtube",
status: status,
privacy_status: "unlisted"
).tap { |s| s.update_columns(updated_at: updated_at) }
end
it "chiude sessioni idle oltre STREAM_SESSION_IDLE_MINUTES (default 30)" do
stale = create_session(status: "idle", updated_at: 45.minutes.ago)
fresh = create_session(status: "idle", updated_at: 10.minutes.ago)
described_class.new.perform
expect(stale.reload.status).to eq("ended")
expect(fresh.reload.status).to eq("idle")
expect(stale.stream_events.where(event_type: "error").exists?).to eq(true)
end
it "chiude connecting/reconnecting stale oltre STREAM_SESSION_STALE_HOURS (default 6)" do
stale = create_session(status: "connecting", updated_at: 7.hours.ago)
fresh = create_session(status: "connecting", updated_at: 1.hour.ago)
described_class.new.perform
expect(stale.reload.status).to eq("ended")
expect(fresh.reload.status).to eq("connecting")
end
it "non tocca sessioni live" do
live = create_session(status: "live", updated_at: 2.hours.ago)
live.update_columns(started_at: 2.hours.ago)
described_class.new.perform
expect(live.reload.status).to eq("live")
end
end
+2
View File
@@ -112,6 +112,8 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0
STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2 STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2
STREAM_AUTOSCALE_WARM_SPARE=1 STREAM_AUTOSCALE_WARM_SPARE=1
STREAM_AUTOSCALE_IDLE_MINUTES=30 STREAM_AUTOSCALE_IDLE_MINUTES=30
STREAM_SESSION_IDLE_MINUTES=30
STREAM_SESSION_STALE_HOURS=6
STREAM_AUTOSCALE_MAX_NODES=12 STREAM_AUTOSCALE_MAX_NODES=12
STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015 STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015
STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150 STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150
+3
View File
@@ -148,6 +148,9 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0
STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2 STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2
STREAM_AUTOSCALE_WARM_SPARE=1 STREAM_AUTOSCALE_WARM_SPARE=1
STREAM_AUTOSCALE_IDLE_MINUTES=30 STREAM_AUTOSCALE_IDLE_MINUTES=30
# Sessioni idle (wizard mai partito) / connecting-reconnecting stale: liberano slot MediaMTX
STREAM_SESSION_IDLE_MINUTES=30
STREAM_SESSION_STALE_HOURS=6
# Fase A (qualità-first): soft ≈ 4 + 12×6 = 76. Fasi successive (dopo misure): # Fase A (qualità-first): soft ≈ 4 + 12×6 = 76. Fasi successive (dopo misure):
# B: MAX_NODES=22 BUDGET=250 → ~136 # B: MAX_NODES=22 BUDGET=250 → ~136
# C: MAX_NODES=33 BUDGET=360 → ~202 # C: MAX_NODES=33 BUDGET=360 → ~202
+1
View File
@@ -33,6 +33,7 @@ CRON_BLOCK="${MARKER}
15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:aggregate >> ${LOG_DIR}/cron-analytics.log 2>&1 15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:aggregate >> ${LOG_DIR}/cron-analytics.log 2>&1
20 4 * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:purge >> ${LOG_DIR}/cron-analytics.log 2>&1 20 4 * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:purge >> ${LOG_DIR}/cron-analytics.log 2>&1
*/15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:night_cloud_sweep >> ${LOG_DIR}/cron-night-sweep.log 2>&1 */15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:night_cloud_sweep >> ${LOG_DIR}/cron-night-sweep.log 2>&1
*/10 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:cleanup_expired_sessions >> ${LOG_DIR}/cron-session-cleanup.log 2>&1
*/10 * * * * mkdir -p ${LOG_DIR} && /bin/bash ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1 */10 * * * * mkdir -p ${LOG_DIR} && /bin/bash ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1
" "