Introduce JF dedicated env vars for running in a distributed mode

Dockerfile

@@ -32,6 +32,8 @@ ENV MIX_ENV=prod
 # but not deps fetching
 # * any changes in the `config/runtime.exs` won't trigger 
 # anything
+# * any changes in rel directory should only trigger
+# making a new release
 COPY mix.exs mix.lock ./
 RUN mix deps.get --only $MIX_ENV
 
@@ -43,6 +45,8 @@ RUN mix compile
 
 COPY config/runtime.exs config/
 
+COPY rel rel
+
 RUN mix release
 
 FROM alpine:3.17 AS app

config/runtime.exs

@@ -11,18 +11,6 @@ alias Jellyfish.ConfigReader
 config :ex_dtls, impl: :nif
 config :opentelemetry, traces_exporter: :none
 
-nodes = ConfigReader.read_nodes("JF_NODES")
-
-if nodes do
-  config :libcluster,
-    topologies: [
-      epmd_cluster: [
-        strategy: Cluster.Strategy.Epmd,
-        config: [hosts: nodes]
-      ]
-    ]
-end
-
 prod? = config_env() == :prod
 
 ip =
@@ -57,7 +45,8 @@ config :jellyfish,
   output_base_path: System.get_env("JF_OUTPUT_BASE_PATH", "jellyfish_output") |> Path.expand(),
   address: "#{host}",
   metrics_ip: ConfigReader.read_ip("JF_METRICS_IP") || {127, 0, 0, 1},
-  metrics_port: ConfigReader.read_port("JF_METRICS_PORT") || 9568
+  metrics_port: ConfigReader.read_port("JF_METRICS_PORT") || 9568,
+  dist_config: ConfigReader.read_dist_config()
 
 case System.get_env("JF_SERVER_API_TOKEN") do
   nil when prod? == true ->

docker-compose.yaml

@@ -3,9 +3,10 @@ version: "3"
 x-jellyfish-template: &jellyfish-template
   build: .
   environment: &jellyfish-environment
-    ERLANG_COOKIE: "panuozzo-pollo-e-pancetta"
     JF_SERVER_API_TOKEN: "development"
+    JF_DIST_ENABLED: "true"
     JF_NODES: "app@app1 app@app2"
+    JF_COOKIE: "panuozzo-pollo-e-pancetta"
   networks:
     - net1
   restart: on-failure
@@ -34,21 +35,19 @@ services:
     <<: *jellyfish-template
     environment:
       <<: *jellyfish-environment
-      RELEASE_NODE: app@app1
-      NODE_NAME: app@app1
       JF_HOST: "localhost:4001"
       JF_PORT: 4001
+      JF_NODE_NAME: app@app1
     ports:
       - 4001:4001
 
   app2:
     <<: *jellyfish-template
     environment:
       <<: *jellyfish-environment
-      RELEASE_NODE: app@app2
-      NODE_NAME: app@app2
       JF_HOST: "localhost:4002"
       JF_PORT: 4002
+      JF_NODE_NAME: app@app2
     ports:
       - 4002:4002
 

lib/jellyfish/application.ex

@@ -5,39 +5,39 @@ defmodule Jellyfish.Application do
 
   use Application
 
+  require Logger
+
   @impl true
   def start(_type, _args) do
     scrape_interval = Application.fetch_env!(:jellyfish, :metrics_scrape_interval)
+    dist_config = Application.fetch_env!(:jellyfish, :dist_config)
 
-    topologies = Application.get_env(:libcluster, :topologies) || []
-
-    children = [
-      {Phoenix.PubSub, name: Jellyfish.PubSub},
-      {Membrane.TelemetryMetrics.Reporter,
-       [
-         metrics: Membrane.RTC.Engine.Endpoint.WebRTC.Metrics.metrics(),
-         name: JellyfishMetricsReporter
-       ]},
-      {Jellyfish.MetricsScraper, scrape_interval},
-      JellyfishWeb.Endpoint,
-      # Start the RoomService
-      Jellyfish.RoomService,
-      {Registry, keys: :unique, name: Jellyfish.RoomRegistry},
-      {Registry, keys: :unique, name: Jellyfish.RequestHandlerRegistry},
-      # Start the Telemetry supervisor (must be started after Jellyfish.RoomRegistry)
-      JellyfishWeb.Telemetry,
-      {Task.Supervisor, name: Jellyfish.TaskSupervisor}
-    ]
+    children =
+      [
+        {Phoenix.PubSub, name: Jellyfish.PubSub},
+        {Membrane.TelemetryMetrics.Reporter,
+         [
+           metrics: Membrane.RTC.Engine.Endpoint.WebRTC.Metrics.metrics(),
+           name: JellyfishMetricsReporter
+         ]},
+        {Jellyfish.MetricsScraper, scrape_interval},
+        JellyfishWeb.Endpoint,
+        # Start the RoomService
+        Jellyfish.RoomService,
+        {Registry, keys: :unique, name: Jellyfish.RoomRegistry},
+        {Registry, keys: :unique, name: Jellyfish.RequestHandlerRegistry},
+        # Start the Telemetry supervisor (must be started after Jellyfish.RoomRegistry)
+        JellyfishWeb.Telemetry,
+        {Task.Supervisor, name: Jellyfish.TaskSupervisor}
+      ] ++
+        if dist_config[:enabled] do
+          config_distribution(dist_config)
+        else
+          []
+        end
 
     :ets.new(:rooms_to_tables, [:public, :set, :named_table])
 
-    children =
-      if topologies == [] do
-        children
-      else
-        [{Cluster.Supervisor, [topologies, [name: Jellyfish.ClusterSupervisor]]} | children]
-      end
-
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Jellyfish.Supervisor]
@@ -51,4 +51,50 @@ defmodule Jellyfish.Application do
     JellyfishWeb.Endpoint.config_change(changed, removed)
     :ok
   end
+
+  defp config_distribution(dist_config) do
+    ensure_epmd_started!()
+
+    # Release always starts in a distributed mode
+    # so we have to start a node only in development.
+    # See env.sh.eex for more information.
+    unless Node.alive?() do
+      case Node.start(dist_config[:node_name]) do
+        {:ok, _} ->
+          :ok
+
+        {:error, reason} ->
+          raise "Couldn't start Jellyfish node, reason: #{inspect(reason)}"
+      end
+
+      Node.set_cookie(dist_config[:cookie])
+    end
+
+    topologies = [
+      epmd_cluster: [
+        strategy: Cluster.Strategy.Epmd,
+        config: [hosts: dist_config[:nodes]]
+      ]
+    ]
+
+    [{Cluster.Supervisor, [topologies, [name: Jellyfish.ClusterSupervisor]]}]
+  end
+
+  defp ensure_epmd_started!() do
+    case System.cmd("epmd", ["-daemon"]) do
+      {_, 0} ->
+        :ok
+
+      _other ->
+        raise """
+        Couldn't start epmd daemon.
+        Epmd is required to run Jellyfish in a distributed mode.
+        You can try to start it manually with:
+          
+          epmd -daemon
+
+        and run Jellyfish again.
+        """
+    end
+  end
 end

lib/jellyfish/config_reader.ex

@@ -1,6 +1,8 @@
 defmodule Jellyfish.ConfigReader do
   @moduledoc false
 
+  require Logger
+
   def read_port_range(env) do
     if value = System.get_env(env) do
       with [str1, str2] <- String.split(value, "-"),
@@ -48,19 +50,55 @@ defmodule Jellyfish.ConfigReader do
     end
   end
 
-  def read_nodes(env) do
-    value = System.get_env(env)
-
-    if value not in ["", nil] do
-      value
-      |> String.split(" ", trim: true)
-      |> Enum.map(&String.to_atom(&1))
-    end
-  end
-
   def read_boolean(env) do
     if value = System.get_env(env) do
       String.downcase(value) not in ["false", "f", "0"]
     end
   end
+
+  def read_dist_config() do
+    if read_boolean("JF_DIST_ENABLED") do
+      node_name_value = System.get_env("JF_NODE_NAME")
+      cookie_value = System.get_env("JF_COOKIE")
+      nodes_value = System.get_env("JF_NODES") || ""
+
+      unset_var =
+        [{"JF_NODE_NAME", node_name_value}, {"JF_COOKIE", cookie_value}]
+        |> Enum.find(fn {_env_name, env_val} -> is_nil(env_val) end)
+
+      if unset_var do
+        {unset_var_name, _} = unset_var
+
+        raise """
+        JF_DIST_ENABLED has been set but #{unset_var_name} remains unset.
+        JF_DIST_ENABLED requires following env vars to be also set: JF_NODE_NAME JF_COOKIE.
+        """
+      end
+
+      node_name = parse_node_name(node_name_value)
+      cookie = parse_cookie(cookie_value)
+      nodes = parse_nodes(nodes_value)
+
+      if nodes == [] do
+        Logger.warning("""
+        JF_DIST_ENABLED has been set but JF_NODES remains unset.
+        Make sure that at least one of your Jellyfish instances
+        has JF_NODES set.
+        """)
+      end
+
+      [enabled: true, node_name: node_name, cookie: cookie, nodes: nodes]
+    else
+      [enabled: false, node_name: nil, cookie: nil, nodes: []]
+    end
+  end
+
+  defp parse_node_name(node_name), do: String.to_atom(node_name)
+  defp parse_cookie(cookie_value), do: String.to_atom(cookie_value)
+
+  defp parse_nodes(nodes_value) do
+    nodes_value
+    |> String.split(" ", trim: true)
+    |> Enum.map(&String.to_atom(&1))
+  end
 end

rel/env.sh.eex

@@ -0,0 +1,7 @@
+# release always starts in a distributed mode
+# but to provide unified way of configuring Jellyfish
+# distribution both in the local and production environment
+# we introduce our own env vars and use them to
+# set RELASE_NODE and RELASE_COOKIE
+export RELEASE_NODE=${JF_NODE_NAME:-${RELEASE_NODE:-jellyfish}}
+export RELEASE_COOKIE=${JF_COOKIE:-${RELEASE_COOKIE:-"$(cat "$RELEASE_ROOT/releases/COOKIE")"}}

test/jellyfish/config_reader_test.exs

@@ -1,25 +1,32 @@
 defmodule Jellyfish.ConfigReaderTest do
-  use ExUnit.Case, async: true
+  use ExUnit.Case
+
+  # run this test synchronously as we use
+  # official env vars in read_dist_config test
 
   alias Jellyfish.ConfigReader
 
   defmacrop with_env(env, do: body) do
-    # get current env value, 
+    # get current env(s) value(s), 
     # execute test code,
-    # put back original env value
+    # put back original env(s) value(s)
     #
     # if env was not set, we have
     # to call System.delete_env as
     # System.put_env does not accept `nil`
     quote do
-      old = System.get_env(unquote(env))
+      envs = List.wrap(unquote(env))
+      old_envs = Enum.map(envs, fn env_name -> {env_name, System.get_env(env_name)} end)
+
       unquote(body)
 
-      if old do
-        System.put_env(unquote(env), old)
-      else
-        System.delete_env(unquote(env))
-      end
+      Enum.each(old_envs, fn {env_name, env_value} ->
+        if env_value do
+          System.put_env(env_name, env_value)
+        else
+          System.delete_env(env_name)
+        end
+      end)
     end
   end
 
@@ -81,14 +88,36 @@ defmodule Jellyfish.ConfigReaderTest do
     end
   end
 
-  test "read_nodes/1" do
-    env_name = "JF_CONF_READER_TEST_NODES"
+  test "read_dist_config/0" do
+    with_env ["JF_DIST_ENABLED", "JF_COOKIE", "JF_NODE_NAME", "JF_NODES"] do
+      assert ConfigReader.read_dist_config() == [
+               enabled: false,
+               node_name: nil,
+               cookie: nil,
+               nodes: []
+             ]
 
-    with_env env_name do
-      System.put_env(env_name, "app1@127.0.0.1 app2@127.0.0.2")
-      assert ConfigReader.read_nodes(env_name) == [:"app1@127.0.0.1", :"app2@127.0.0.2"]
-      System.put_env(env_name, "")
-      assert ConfigReader.read_nodes(env_name) == nil
+      System.put_env("JF_DIST_ENABLED", "true")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_COOKIE", "testcookie")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_NODE_NAME", "testnodename@127.0.0.1")
+
+      assert ConfigReader.read_dist_config() == [
+               enabled: true,
+               node_name: :"testnodename@127.0.0.1",
+               cookie: :testcookie,
+               nodes: []
+             ]
+
+      System.put_env("JF_NODES", "testnodename1@127.0.0.1 testnodename2@127.0.0.1")
+
+      assert ConfigReader.read_dist_config() == [
+               enabled: true,
+               node_name: :"testnodename@127.0.0.1",
+               cookie: "testcookie",
+               nodes: [:"testnodename1@127.0.0.1", :"testnodename2@127.0.0.1"]
+             ]
     end
   end
 end