Chiude le sessioni idle abbandonate prima che occupino slot e scalino Hetzner.
Il sweeper ora termina le idle dopo 30 minuti (cron ogni 10), così non restano appese a tenere accesi i warm-spare. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,12 +1,65 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
# Chiude sessioni abbandonate che altrimenti occupano slot MediaMTX e
|
||||||
|
# fanno scattare warm-spare / scale-out Hetzner.
|
||||||
|
#
|
||||||
|
# - idle: wizard aperti e mai partiti (default 30 min)
|
||||||
|
# - connecting / reconnecting: hang senza go_live (default 6 h)
|
||||||
class CleanupExpiredSessionsJob
|
class CleanupExpiredSessionsJob
|
||||||
include Sidekiq::Job
|
include Sidekiq::Job
|
||||||
|
|
||||||
def perform
|
def perform
|
||||||
|
closed_idle = sweep_idle!
|
||||||
|
closed_stale = sweep_stale_connect!
|
||||||
|
Rails.logger.info(
|
||||||
|
"[CleanupExpiredSessionsJob] idle_closed=#{closed_idle} stale_closed=#{closed_stale} " \
|
||||||
|
"idle_minutes=#{self.class.idle_minutes} stale_hours=#{self.class.stale_hours}"
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.idle_minutes
|
||||||
|
ENV.fetch("STREAM_SESSION_IDLE_MINUTES", "30").to_i
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.stale_hours
|
||||||
|
ENV.fetch("STREAM_SESSION_STALE_HOURS", "6").to_i
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def sweep_idle!
|
||||||
|
cutoff = self.class.idle_minutes.minutes.ago
|
||||||
|
count = 0
|
||||||
|
StreamSession.where(status: "idle").where("updated_at < ?", cutoff).find_each do |session|
|
||||||
|
close!(session, reason: "idle_timeout")
|
||||||
|
count += 1
|
||||||
|
end
|
||||||
|
count
|
||||||
|
end
|
||||||
|
|
||||||
|
def sweep_stale_connect!
|
||||||
|
cutoff = self.class.stale_hours.hours.ago
|
||||||
|
count = 0
|
||||||
StreamSession.where(status: %w[connecting reconnecting])
|
StreamSession.where(status: %w[connecting reconnecting])
|
||||||
.where("updated_at < ?", 6.hours.ago)
|
.where("updated_at < ?", cutoff)
|
||||||
.find_each do |session|
|
.find_each do |session|
|
||||||
session.fail! if session.may_fail?
|
close!(session, reason: "stale_#{session.status}")
|
||||||
Mediamtx::Client.for_session(session).delete_path(session)
|
count += 1
|
||||||
end
|
end
|
||||||
|
count
|
||||||
|
end
|
||||||
|
|
||||||
|
def close!(session, reason:)
|
||||||
|
session.stream_events.create!(
|
||||||
|
event_type: "error",
|
||||||
|
metadata: { reason: reason },
|
||||||
|
occurred_at: Time.current
|
||||||
|
)
|
||||||
|
Sessions::Stop.new(session).call
|
||||||
|
rescue StandardError => e
|
||||||
|
Rails.logger.warn(
|
||||||
|
"[CleanupExpiredSessionsJob] session=#{session.id} reason=#{reason} " \
|
||||||
|
"#{e.class}: #{e.message}"
|
||||||
|
)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -1,2 +1,3 @@
|
|||||||
# Schedule CleanupExpiredSessionsJob hourly via host cron or Kamal:
|
# CleanupExpiredSessionsJob: host cron via infra/scripts/install_production_cron.sh
|
||||||
# 0 * * * * cd /app && bundle exec rails runner "CleanupExpiredSessionsJob.perform_async"
|
# */10 * * * * ... streams:cleanup_expired_sessions
|
||||||
|
# Env: STREAM_SESSION_IDLE_MINUTES (default 30), STREAM_SESSION_STALE_HOURS (default 6)
|
||||||
|
|||||||
@@ -7,6 +7,11 @@ namespace :streams do
|
|||||||
puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}"
|
puts "skipped=#{result.skipped} error=#{result.error} actions=#{result.actions.inspect}"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
desc "Chiude sessioni idle/stale che occupano slot (STREAM_SESSION_IDLE_MINUTES, default 30)"
|
||||||
|
task cleanup_expired_sessions: :environment do
|
||||||
|
CleanupExpiredSessionsJob.new.perform
|
||||||
|
end
|
||||||
|
|
||||||
namespace :nodes do
|
namespace :nodes do
|
||||||
desc "Assicura il nodo home dagli ENV MediaMTX"
|
desc "Assicura il nodo home dagli ENV MediaMTX"
|
||||||
task ensure_home: :environment do
|
task ensure_home: :environment do
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require "rails_helper"
|
||||||
|
|
||||||
|
RSpec.describe CleanupExpiredSessionsJob do
|
||||||
|
let(:user) { User.create!(email: "idle-clean@test.it", name: "Idle", password: "Password123", role: "coach") }
|
||||||
|
let(:club) { Club.create!(name: "Club Idle", sport: "volleyball") }
|
||||||
|
let(:team) { club.teams.create!(name: "Team Idle", sport: "volleyball") }
|
||||||
|
let(:match) { team.matches.create!(opponent_name: "Opp", sport: "volleyball") }
|
||||||
|
|
||||||
|
before do
|
||||||
|
club.club_memberships.create!(user: user, role: "owner")
|
||||||
|
allow(SessionChannel).to receive(:broadcast_message)
|
||||||
|
allow(Streams::YoutubeRelay).to receive(:stop)
|
||||||
|
client = instance_double(Mediamtx::Client, delete_path: true)
|
||||||
|
allow(Mediamtx::Client).to receive(:for_session).and_return(client)
|
||||||
|
allow(Recordings::FinalizeSession).to receive(:new).and_return(
|
||||||
|
instance_double(Recordings::FinalizeSession, call: nil)
|
||||||
|
)
|
||||||
|
allow(Tournaments::CaptureStreamResult).to receive(:call)
|
||||||
|
end
|
||||||
|
|
||||||
|
def create_session(status:, updated_at:)
|
||||||
|
StreamSession.create!(
|
||||||
|
match: match,
|
||||||
|
user: user,
|
||||||
|
platform: "youtube",
|
||||||
|
status: status,
|
||||||
|
privacy_status: "unlisted"
|
||||||
|
).tap { |s| s.update_columns(updated_at: updated_at) }
|
||||||
|
end
|
||||||
|
|
||||||
|
it "chiude sessioni idle oltre STREAM_SESSION_IDLE_MINUTES (default 30)" do
|
||||||
|
stale = create_session(status: "idle", updated_at: 45.minutes.ago)
|
||||||
|
fresh = create_session(status: "idle", updated_at: 10.minutes.ago)
|
||||||
|
|
||||||
|
described_class.new.perform
|
||||||
|
|
||||||
|
expect(stale.reload.status).to eq("ended")
|
||||||
|
expect(fresh.reload.status).to eq("idle")
|
||||||
|
expect(stale.stream_events.where(event_type: "error").exists?).to eq(true)
|
||||||
|
end
|
||||||
|
|
||||||
|
it "chiude connecting/reconnecting stale oltre STREAM_SESSION_STALE_HOURS (default 6)" do
|
||||||
|
stale = create_session(status: "connecting", updated_at: 7.hours.ago)
|
||||||
|
fresh = create_session(status: "connecting", updated_at: 1.hour.ago)
|
||||||
|
|
||||||
|
described_class.new.perform
|
||||||
|
|
||||||
|
expect(stale.reload.status).to eq("ended")
|
||||||
|
expect(fresh.reload.status).to eq("connecting")
|
||||||
|
end
|
||||||
|
|
||||||
|
it "non tocca sessioni live" do
|
||||||
|
live = create_session(status: "live", updated_at: 2.hours.ago)
|
||||||
|
live.update_columns(started_at: 2.hours.ago)
|
||||||
|
|
||||||
|
described_class.new.perform
|
||||||
|
|
||||||
|
expect(live.reload.status).to eq("live")
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -112,6 +112,8 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0
|
|||||||
STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2
|
STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2
|
||||||
STREAM_AUTOSCALE_WARM_SPARE=1
|
STREAM_AUTOSCALE_WARM_SPARE=1
|
||||||
STREAM_AUTOSCALE_IDLE_MINUTES=30
|
STREAM_AUTOSCALE_IDLE_MINUTES=30
|
||||||
|
STREAM_SESSION_IDLE_MINUTES=30
|
||||||
|
STREAM_SESSION_STALE_HOURS=6
|
||||||
STREAM_AUTOSCALE_MAX_NODES=12
|
STREAM_AUTOSCALE_MAX_NODES=12
|
||||||
STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015
|
STREAM_AUTOSCALE_NODE_EUR_PER_HOUR=0.015
|
||||||
STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150
|
STREAM_AUTOSCALE_MONTHLY_BUDGET_EUR=150
|
||||||
|
|||||||
@@ -148,6 +148,9 @@ STREAM_AUTOSCALE_ALLOW_CLOUD=0
|
|||||||
STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2
|
STREAM_AUTOSCALE_SOFT_FREE_SLOTS=2
|
||||||
STREAM_AUTOSCALE_WARM_SPARE=1
|
STREAM_AUTOSCALE_WARM_SPARE=1
|
||||||
STREAM_AUTOSCALE_IDLE_MINUTES=30
|
STREAM_AUTOSCALE_IDLE_MINUTES=30
|
||||||
|
# Sessioni idle (wizard mai partito) / connecting-reconnecting stale: liberano slot MediaMTX
|
||||||
|
STREAM_SESSION_IDLE_MINUTES=30
|
||||||
|
STREAM_SESSION_STALE_HOURS=6
|
||||||
# Fase A (qualità-first): soft ≈ 4 + 12×6 = 76. Fasi successive (dopo misure):
|
# Fase A (qualità-first): soft ≈ 4 + 12×6 = 76. Fasi successive (dopo misure):
|
||||||
# B: MAX_NODES=22 BUDGET=250 → ~136
|
# B: MAX_NODES=22 BUDGET=250 → ~136
|
||||||
# C: MAX_NODES=33 BUDGET=360 → ~202
|
# C: MAX_NODES=33 BUDGET=360 → ~202
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ CRON_BLOCK="${MARKER}
|
|||||||
15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:aggregate >> ${LOG_DIR}/cron-analytics.log 2>&1
|
15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:aggregate >> ${LOG_DIR}/cron-analytics.log 2>&1
|
||||||
20 4 * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:purge >> ${LOG_DIR}/cron-analytics.log 2>&1
|
20 4 * * * mkdir -p ${LOG_DIR} && ${RUNNER} analytics:purge >> ${LOG_DIR}/cron-analytics.log 2>&1
|
||||||
*/15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:night_cloud_sweep >> ${LOG_DIR}/cron-night-sweep.log 2>&1
|
*/15 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:night_cloud_sweep >> ${LOG_DIR}/cron-night-sweep.log 2>&1
|
||||||
|
*/10 * * * * mkdir -p ${LOG_DIR} && ${RUNNER} streams:cleanup_expired_sessions >> ${LOG_DIR}/cron-session-cleanup.log 2>&1
|
||||||
*/10 * * * * mkdir -p ${LOG_DIR} && /bin/bash ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1
|
*/10 * * * * mkdir -p ${LOG_DIR} && /bin/bash ${SCAN_LOGS} >> ${OPS_LOG_FILE} 2>&1
|
||||||
"
|
"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user