Fix race create sessione su CPX: ready solo dopo MediaMTX.

I nodi cloud restano in provisioning finché :9997 risponde; retry su create_path e 503 retryable se l’ingest è ancora irraggiungibile. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-21 14:06:47 +02:00
parent 70fccc493e
commit b87a0f7bc0
12 changed files with 372 additions and 3 deletions
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe "API session create ingest unavailable", type: :request do
+  let!(:user) do
+    User.create!(email: "ingest-#{SecureRandom.hex(4)}@example.com", name: "Coach", password: "Password123", role: "coach")
+  end
+  let!(:club) { Club.create!(name: "IngestClub", sport: "volleyball") }
+  let!(:membership) { club.club_memberships.create!(user: user, role: "owner") }
+  let!(:team) { club.teams.create!(name: "Tigers", sport: "volleyball", slug: "ingest-tigers-#{SecureRandom.hex(3)}") }
+  let!(:match) { team.matches.create!(opponent_name: "Opp", scheduled_at: 1.hour.from_now) }
+
+  def auth_headers
+    post "/api/v1/auth/login", params: { email: user.email, password: "Password123" }
+    token = response.parsed_body["access_token"]
+    { "Authorization" => "Bearer #{token}", "Content-Type" => "application/json" }
+  end
+
+  before do
+    plan = Plan.find_or_initialize_by(slug: "premium_full")
+    plan.name ||= "Premium Full"
+    plan.features = (plan.features || {}).merge(
+      "platforms" => %w[matchlivetv youtube],
+      "youtube_enabled" => true,
+      "concurrent_streams_limit" => 10,
+      "recordings_enabled" => true,
+      "recording_days" => 90
+    )
+    plan.save!
+    club.create_subscription!(plan: plan, status: "active") if club.subscription.blank?
+
+    allow_any_instance_of(Teams::Entitlements).to receive(:assert_can_stream_on!)
+    allow_any_instance_of(Teams::Entitlements).to receive(:assert_concurrent_stream!)
+    allow(Streams::CoverSlateEnsurer).to receive(:ensure_for!)
+    allow(Streams::SlateDistributor).to receive(:ensure_for!)
+    Streams::NodeRegistry.ensure_home_from_env!
+  end
+
+  it "returns 503 when MediaMTX connection fails after retries" do
+    allow_any_instance_of(Mediamtx::Client).to receive(:create_path)
+      .and_raise(Faraday::ConnectionFailed.new("Connection refused"))
+
+    post "/api/v1/matches/#{match.id}/sessions",
+         params: { platform: "matchlivetv", privacy_status: "private" }.to_json,
+         headers: auth_headers
+
+    expect(response).to have_http_status(:service_unavailable)
+    body = response.parsed_body
+    expect(body["error_code"]).to eq("stream_ingest_unavailable")
+  end
+end
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Mediamtx::Client do
+  describe "#create_path" do
+    let(:session) do
+      user = User.create!(email: "mtx-#{SecureRandom.hex(4)}@example.com", name: "M", password: "Password123", role: "coach")
+      club = Club.create!(name: "MtxClub-#{SecureRandom.hex(3)}", sport: "volleyball")
+      team = club.teams.create!(name: "T", sport: "volleyball", slug: "mtx-#{SecureRandom.hex(4)}")
+      match = team.matches.create!(opponent_name: "X", scheduled_at: 1.hour.from_now)
+      StreamSession.create!(match: match, user: user, platform: "matchlivetv", status: "idle")
+    end
+
+    it "retries Faraday connection failures then succeeds" do
+      ENV["MEDIAMTX_CREATE_RETRIES"] = "3"
+      ENV["MEDIAMTX_CREATE_RETRY_BASE_SECS"] = "0"
+
+      client = described_class.new(base_url: "http://mtx.test:9997")
+      conn = instance_double(Faraday::Connection)
+      client.instance_variable_set(:@conn, conn)
+
+      fail_once = Faraday::ConnectionFailed.new("Connection refused")
+      ok = instance_double(Faraday::Response, success?: true, status: 200, body: {})
+
+      expect(conn).to receive(:post).once.and_raise(fail_once)
+      expect(conn).to receive(:post).once.and_return(ok)
+
+      expect(client.create_path(session)).to eq(true)
+    ensure
+      ENV.delete("MEDIAMTX_CREATE_RETRIES")
+      ENV.delete("MEDIAMTX_CREATE_RETRY_BASE_SECS")
+    end
+
+    it "raises after exhausting connection retries" do
+      ENV["MEDIAMTX_CREATE_RETRIES"] = "2"
+      ENV["MEDIAMTX_CREATE_RETRY_BASE_SECS"] = "0"
+
+      client = described_class.new(base_url: "http://mtx.test:9997")
+      conn = instance_double(Faraday::Connection)
+      client.instance_variable_set(:@conn, conn)
+      allow(conn).to receive(:post).and_raise(Faraday::ConnectionFailed.new("Connection refused"))
+
+      expect { client.create_path(session) }.to raise_error(Faraday::ConnectionFailed)
+    ensure
+      ENV.delete("MEDIAMTX_CREATE_RETRIES")
+      ENV.delete("MEDIAMTX_CREATE_RETRY_BASE_SECS")
+    end
+  end
+end
@@ -224,4 +224,41 @@ RSpec.describe Streams::Autoscaler do
      expect(described_class.within_budget?(1)).to eq(false)
    end
  end
+
+  it "promotes provisioning nodes when MediaMTX becomes reachable" do
+    with_env(
+      "STREAM_AUTOSCALE_ENABLED" => "1",
+      "STREAM_AUTOSCALE_WARM_SPARE" => "0",
+      "STREAM_AUTOSCALE_SOFT_FREE_SLOTS" => "0",
+      "STREAM_AUTOSCALE_KIND" => "lab",
+      "MEDIAMTX_API_URL" => "http://mtx-home:9997",
+      "MEDIAMTX_RTMP_URL" => "rtmp://home.example:1935",
+      "HLS_PUBLIC_URL" => "https://home.example/hls"
+    ) do
+      Streams::NodeRegistry.ensure_home_from_env!
+      node = StreamNode.create!(
+        slug: "ingest-lab-prov",
+        hostname: "prov.lab",
+        role: "lab",
+        status: "provisioning",
+        provider: "local",
+        rtmp_base_url: "rtmp://h:1935",
+        hls_base_url: "https://h/hls",
+        api_base_url: "http://h:9997",
+        max_publishers: 2,
+        max_relays: 2
+      )
+      allow(Streams::NodeHealth).to receive(:promote_if_healthy!) do |n|
+        n.update!(status: "ready", last_health_at: Time.current)
+        true
+      end
+
+      provisioner = instance_double(Streams::NodeProvisioner)
+      expect(provisioner).not_to receive(:provision_lab!)
+
+      result = described_class.reconcile!(provisioner: provisioner)
+      expect(result.actions).to include(:"ready_ingest-lab-prov")
+      expect(node.reload.status).to eq("ready")
+    end
+  end
 end
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe Streams::NodeHealth do
+  let!(:node) do
+    StreamNode.create!(
+      slug: "ingest-health-01",
+      hostname: "ingest-health-01.mltv-stream.net",
+      role: "cloud",
+      status: "provisioning",
+      provider: "hetzner",
+      rtmp_base_url: "rtmp://h:1935",
+      hls_base_url: "https://h/hls",
+      api_base_url: "http://203.0.113.10:9997",
+      max_publishers: 4,
+      max_relays: 4
+    )
+  end
+
+  after { node.destroy }
+
+  it "promotes provisioning node when MediaMTX is reachable" do
+    client = instance_double(Mediamtx::Client, reachable?: true)
+    allow(Mediamtx::Client).to receive(:new).with(base_url: node.api_base_url).and_return(client)
+
+    expect(described_class.promote_if_healthy!(node)).to eq(true)
+    expect(node.reload.status).to eq("ready")
+    expect(node.last_health_at).to be_present
+  end
+
+  it "does not promote when MediaMTX is down" do
+    client = instance_double(Mediamtx::Client, reachable?: false)
+    allow(Mediamtx::Client).to receive(:new).with(base_url: node.api_base_url).and_return(client)
+
+    expect(described_class.promote_if_healthy!(node)).to eq(false)
+    expect(node.reload.status).to eq("provisioning")
+  end
+end
@@ -24,11 +24,15 @@ RSpec.describe "Streams::NodeProvisioner cloud" do
    ENV["STREAM_CLOUD_DNS_SUFFIX"] = "mltv-stream.net"
    ENV["STREAM_CLOUD_MAX_PUBLISHERS"] = "4"
    ENV["STREAM_CLOUD_PUBLIC_CONTROL"] = "0"
+    ENV["STREAM_NODE_READY_TIMEOUT_SECS"] = "0"
+
+    allow(Streams::NodeHealth).to receive(:promote_if_healthy!).and_return(false)

    node = Streams::NodeProvisioner.new(cloud: cloud, dns: dns).provision_cloud!
    expect(node.slug).to eq("ingest-01")
    expect(node.role).to eq("cloud")
    expect(node.provider).to eq("hetzner")
+    expect(node.status).to eq("provisioning")
    expect(node.hostname).to eq("ingest-01.mltv-stream.net")
    expect(node.rtmp_base_url).to eq("rtmp://ingest-01.mltv-stream.net:1935")
    expect(node.api_base_url).to eq("http://10.0.0.9:9997")
@@ -39,6 +43,45 @@ RSpec.describe "Streams::NodeProvisioner cloud" do
    %w[
      MEDIAMTX_API_URL MEDIAMTX_RTMP_URL HLS_PUBLIC_URL
      STREAM_CLOUD_DNS_SUFFIX STREAM_CLOUD_MAX_PUBLISHERS STREAM_CLOUD_PUBLIC_CONTROL
+      STREAM_NODE_READY_TIMEOUT_SECS
+    ].each { |k| ENV.delete(k) }
+  end
+
+  it "marks cloud node ready when MediaMTX answers during wait" do
+    cloud = instance_double(
+      Streams::CloudProviders::Hetzner,
+      create_node: Streams::CloudProviders::Instance.new(
+        id: "100",
+        name: "mltv-stream-ingest-01",
+        public_ip: "49.13.9.9",
+        private_ip: "10.0.0.9",
+        status: "running",
+        raw: {}
+      )
+    )
+    dns = instance_double(Streams::DnsProviders::Hetzner)
+    allow(dns).to receive(:upsert_a)
+
+    ENV["MEDIAMTX_API_URL"] = "http://mtx-home:9997"
+    ENV["MEDIAMTX_RTMP_URL"] = "rtmp://home.example:1935"
+    ENV["HLS_PUBLIC_URL"] = "https://home.example/hls"
+    ENV["STREAM_CLOUD_DNS_SUFFIX"] = "mltv-stream.net"
+    ENV["STREAM_CLOUD_PUBLIC_CONTROL"] = "0"
+    ENV["STREAM_NODE_READY_TIMEOUT_SECS"] = "5"
+    ENV["STREAM_NODE_READY_POLL_SECS"] = "0.01"
+
+    allow(Streams::NodeHealth).to receive(:promote_if_healthy!) do |node|
+      node.update!(status: "ready", last_health_at: Time.current)
+      true
+    end
+
+    node = Streams::NodeProvisioner.new(cloud: cloud, dns: dns).provision_cloud!
+    expect(node.status).to eq("ready")
+  ensure
+    %w[
+      MEDIAMTX_API_URL MEDIAMTX_RTMP_URL HLS_PUBLIC_URL
+      STREAM_CLOUD_DNS_SUFFIX STREAM_CLOUD_PUBLIC_CONTROL
+      STREAM_NODE_READY_TIMEOUT_SECS STREAM_NODE_READY_POLL_SECS
    ].each { |k| ENV.delete(k) }
  end
 end