Files
MatchLiveTv/backend/app/services/streams/node_provisioner.rb
eminuxandCursor b87a0f7bc0 Fix race create sessione su CPX: ready solo dopo MediaMTX.
I nodi cloud restano in provisioning finché :9997 risponde; retry su create_path e 503 retryable se l’ingest è ancora irraggiungibile.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-21 14:06:47 +02:00

217 lines
7.6 KiB
Ruby

# frozen_string_literal: true
module Streams
# Provisiona / decommissiona nodi stream (lab o cloud) e aggiorna DNS + registry.
class NodeProvisioner
class Error < StandardError; end
class BusyError < Error; end
LAB_DNS_SUFFIX = -> { ENV.fetch("STREAM_LAB_DNS_SUFFIX", "lab.mltv-stream.net") }
CLOUD_DNS_SUFFIX = -> { ENV.fetch("STREAM_CLOUD_DNS_SUFFIX", ENV.fetch("STREAM_DNS_ZONE", "mltv-stream.net")) }
def initialize(cloud: nil, dns: nil)
@cloud = cloud
@dns = dns
end
def provision_lab!(prefix: "ingest-lab")
provision!(
prefix: prefix,
role: "lab",
dns_suffix: LAB_DNS_SUFFIX.call,
cloud: cloud_provider(ENV.fetch("STREAM_CLOUD_PROVIDER", "local_lab")),
dns: dns_provider(ENV.fetch("STREAM_DNS_PROVIDER", "lab")),
max: ENV.fetch("STREAM_LAB_MAX_PUBLISHERS", "2").to_i,
use_node_hostname: ENV["STREAM_LAB_USE_NODE_HOSTNAME"] == "1"
)
end
def provision_cloud!(prefix: "ingest")
provision!(
prefix: prefix,
role: "cloud",
dns_suffix: CLOUD_DNS_SUFFIX.call,
cloud: cloud_provider("hetzner"),
dns: dns_provider("hetzner"),
max: ENV.fetch("STREAM_CLOUD_MAX_PUBLISHERS", "4").to_i,
use_node_hostname: true
)
end
def decommission!(node)
raise Error, "Non si può decommissionare il nodo home" if node.slug == Streams::NodeRegistry::HOME_SLUG
if node.occupying_sessions.exists?
raise BusyError, "Nodo #{node.slug} ha ancora sessioni attive"
end
node.update!(status: "draining")
cloud = cloud_for_node(node)
dns = dns_for_node(node)
cloud.destroy_node(node.provider_instance_id) if node.provider_instance_id.present?
dns.delete_a(node.hostname) if node.hostname.present?
node.destroy!
true
end
def drain!(node)
raise Error, "Non si può mettere in drain il nodo home" if node.slug == Streams::NodeRegistry::HOME_SLUG
node.update!(status: "draining")
node
end
private
def provision!(prefix:, role:, dns_suffix:, cloud:, dns:, max:, use_node_hostname:)
Streams::NodeRegistry.ensure_home_from_env!
home = StreamNode.find_by!(slug: Streams::NodeRegistry::HOME_SLUG)
slug = next_slug(prefix)
hostname = "#{slug}.#{dns_suffix}"
instance = cloud.create_node(
name: "mltv-stream-#{slug}",
labels: { role: "stream-node", env: role == "cloud" ? "prod" : "lab" }
)
ip = instance.public_ip.presence || "127.0.0.1"
private_ip = instance.private_ip.presence || ip
dns.upsert_a(hostname, ip)
simulated = instance.raw.is_a?(Hash) && (instance.raw[:simulated] || instance.raw["simulated"])
urls = urls_for_node(role: role, home: home, hostname: hostname, simulated: simulated,
private_ip: private_ip, public_ip: ip, use_node_hostname: use_node_hostname)
# Simulated/lab che riusa MediaMTX home: subito ready. Cloud reale: provisioning
# finché :9997 risponde (evita allocate → Faraday Connection refused → 500).
initial_status = simulated ? "ready" : "provisioning"
node = StreamNode.create!(
slug: slug,
hostname: hostname,
role: role,
status: initial_status,
provider: provider_name_for(cloud, role: role),
provider_instance_id: instance.id,
rtmp_base_url: urls.fetch(:rtmp_base_url),
hls_base_url: urls.fetch(:hls_base_url),
api_base_url: urls.fetch(:api_base_url),
internal_rtmp_url: urls.fetch(:internal_rtmp_url),
internal_hls_url: urls.fetch(:internal_hls_url),
max_publishers: max,
max_relays: max,
last_health_at: Time.current,
metadata: {
"public_ip" => ip,
"private_ip" => private_ip,
"simulated" => simulated,
"cloud_raw" => instance.raw
}
)
wait_until_mediamtx_ready!(node) unless simulated
node.reload
end
def wait_until_mediamtx_ready!(node, timeout: nil, interval: nil)
timeout ||= ENV.fetch("STREAM_NODE_READY_TIMEOUT_SECS", "180").to_i
interval ||= ENV.fetch("STREAM_NODE_READY_POLL_SECS", "3").to_f
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
loop do
if Streams::NodeHealth.promote_if_healthy!(node)
Rails.logger.info("[Streams::NodeProvisioner] #{node.slug} MediaMTX ready")
return node
end
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
if remaining <= 0
Rails.logger.warn(
"[Streams::NodeProvisioner] #{node.slug} ancora provisioning dopo #{timeout}s " \
"(api=#{node.api_base_url}); autoscaler continuerà a promuovere"
)
return node
end
sleep([interval, remaining].min)
node.reload
end
end
def urls_for_node(role:, home:, hostname:, simulated:, private_ip:, public_ip: nil, use_node_hostname:)
if role == "lab" && ENV["MEDIAMTX_LAB_API_URL"].present?
return {
api_base_url: ENV.fetch("MEDIAMTX_LAB_API_URL"),
internal_rtmp_url: ENV.fetch("MEDIAMTX_LAB_INTERNAL_RTMP_URL", "rtmp://mediamtx_lab:1935"),
internal_hls_url: ENV.fetch("MEDIAMTX_LAB_HLS_URL", "http://mediamtx_lab:8888"),
rtmp_base_url: ENV.fetch("MEDIAMTX_LAB_RTMP_URL", "rtmp://127.0.0.1:11935"),
# HLS pubblico resta sul proxy del sito: il controller instrada al MediaMTX del nodo.
hls_base_url: home.hls_base_url
}
end
control_ip = control_ip_for(role: role, private_ip: private_ip, public_ip: public_ip)
{
api_base_url: simulated ? home.api_base_url : "http://#{control_ip}:9997",
internal_rtmp_url: simulated ? home.internal_rtmp_url : "rtmp://#{control_ip}:1935",
internal_hls_url: simulated ? home.internal_hls_url : "http://#{control_ip}:8888",
rtmp_base_url: use_node_hostname ? "rtmp://#{hostname}:1935" : home.rtmp_base_url,
# Player del sito: proxy Rails/edge. HTTPS diretto sul nodo arriva dopo TLS/Caddy.
hls_base_url: home.hls_base_url
}
end
# Senza WireGuard/network privata Rails deve parlare con l'IPv4 pubblico (API+HLS).
def control_ip_for(role:, private_ip:, public_ip:)
public_ip = public_ip.presence
return private_ip if role != "cloud" || public_ip.blank?
return public_ip if ENV["STREAM_CLOUD_PUBLIC_CONTROL"] == "1"
return public_ip if private_ip.blank? || private_ip == public_ip
private_ip
end
def next_slug(prefix)
used = StreamNode.where("slug LIKE ?", "#{prefix}-%").pluck(:slug)
n = 1
loop do
candidate = format("%s-%02d", prefix, n)
return candidate unless used.include?(candidate)
n += 1
end
end
def cloud_provider(name)
@cloud || Streams::CloudProviders.build(name)
end
def dns_provider(name)
@dns || Streams::DnsProviders.build(name)
end
def provider_name_for(cloud, role: nil)
return "hetzner" if role.to_s == "cloud"
case cloud
when Streams::CloudProviders::ProxmoxLab then "proxmox_lab"
when Streams::CloudProviders::Hetzner then "hetzner"
else "local"
end
end
def cloud_for_node(node)
case node.provider
when "hetzner" then Streams::CloudProviders::Hetzner.new
when "proxmox_lab" then Streams::CloudProviders::ProxmoxLab.new
else Streams::CloudProviders::LocalLab.new
end
end
def dns_for_node(node)
case node.provider
when "hetzner" then Streams::DnsProviders::Hetzner.new
else Streams::DnsProviders::Lab.new
end
end
end
end