From b417a0383de75322470a3772be6c8e4b8e8b6c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Aled=C5=BA?= Date: Wed, 20 Sep 2023 19:04:57 +0200 Subject: [PATCH] Introduce JF dedicated env vars for running in a distributed mode --- Dockerfile | 4 ++ config/runtime.exs | 15 +--- docker-compose.yaml | 9 ++- lib/jellyfish/application.ex | 98 ++++++++++++++++++++------- lib/jellyfish/config_reader.ex | 58 +++++++++++++--- rel/env.sh.eex | 7 ++ test/jellyfish/config_reader_test.exs | 61 ++++++++++++----- 7 files changed, 182 insertions(+), 70 deletions(-) create mode 100644 rel/env.sh.eex diff --git a/Dockerfile b/Dockerfile index b767cb0f..32025196 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,8 @@ ENV MIX_ENV=prod # but not deps fetching # * any changes in the `config/runtime.exs` won't trigger # anything +# * any changes in rel directory should only trigger +# making a new release COPY mix.exs mix.lock ./ RUN mix deps.get --only $MIX_ENV @@ -43,6 +45,8 @@ RUN mix compile COPY config/runtime.exs config/ +COPY rel rel + RUN mix release FROM alpine:3.17 AS app diff --git a/config/runtime.exs b/config/runtime.exs index 9d48fb21..85deace3 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -11,18 +11,6 @@ alias Jellyfish.ConfigReader config :ex_dtls, impl: :nif config :opentelemetry, traces_exporter: :none -nodes = ConfigReader.read_nodes("JF_NODES") - -if nodes do - config :libcluster, - topologies: [ - epmd_cluster: [ - strategy: Cluster.Strategy.Epmd, - config: [hosts: nodes] - ] - ] -end - prod? = config_env() == :prod ip = @@ -57,7 +45,8 @@ config :jellyfish, output_base_path: System.get_env("JF_OUTPUT_BASE_PATH", "jellyfish_output") |> Path.expand(), address: "#{host}", metrics_ip: ConfigReader.read_ip("JF_METRICS_IP") || {127, 0, 0, 1}, - metrics_port: ConfigReader.read_port("JF_METRICS_PORT") || 9568 + metrics_port: ConfigReader.read_port("JF_METRICS_PORT") || 9568, + dist_config: ConfigReader.read_dist_config() case System.get_env("JF_SERVER_API_TOKEN") do nil when prod? == true -> diff --git a/docker-compose.yaml b/docker-compose.yaml index c58b38ff..cf592808 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,9 +3,10 @@ version: "3" x-jellyfish-template: &jellyfish-template build: . environment: &jellyfish-environment - ERLANG_COOKIE: "panuozzo-pollo-e-pancetta" JF_SERVER_API_TOKEN: "development" + JF_DIST_ENABLED: "true" JF_NODES: "app@app1 app@app2" + JF_COOKIE: "panuozzo-pollo-e-pancetta" networks: - net1 restart: on-failure @@ -34,10 +35,9 @@ services: <<: *jellyfish-template environment: <<: *jellyfish-environment - RELEASE_NODE: app@app1 - NODE_NAME: app@app1 JF_HOST: "localhost:4001" JF_PORT: 4001 + JF_NODE_NAME: app@app1 ports: - 4001:4001 @@ -45,10 +45,9 @@ services: <<: *jellyfish-template environment: <<: *jellyfish-environment - RELEASE_NODE: app@app2 - NODE_NAME: app@app2 JF_HOST: "localhost:4002" JF_PORT: 4002 + JF_NODE_NAME: app@app2 ports: - 4002:4002 diff --git a/lib/jellyfish/application.ex b/lib/jellyfish/application.ex index adc89507..a9eb469c 100644 --- a/lib/jellyfish/application.ex +++ b/lib/jellyfish/application.ex @@ -5,39 +5,39 @@ defmodule Jellyfish.Application do use Application + require Logger + @impl true def start(_type, _args) do scrape_interval = Application.fetch_env!(:jellyfish, :metrics_scrape_interval) + dist_config = Application.fetch_env!(:jellyfish, :dist_config) - topologies = Application.get_env(:libcluster, :topologies) || [] - - children = [ - {Phoenix.PubSub, name: Jellyfish.PubSub}, - {Membrane.TelemetryMetrics.Reporter, - [ - metrics: Membrane.RTC.Engine.Endpoint.WebRTC.Metrics.metrics(), - name: JellyfishMetricsReporter - ]}, - {Jellyfish.MetricsScraper, scrape_interval}, - JellyfishWeb.Endpoint, - # Start the RoomService - Jellyfish.RoomService, - {Registry, keys: :unique, name: Jellyfish.RoomRegistry}, - {Registry, keys: :unique, name: Jellyfish.RequestHandlerRegistry}, - # Start the Telemetry supervisor (must be started after Jellyfish.RoomRegistry) - JellyfishWeb.Telemetry, - {Task.Supervisor, name: Jellyfish.TaskSupervisor} - ] + children = + [ + {Phoenix.PubSub, name: Jellyfish.PubSub}, + {Membrane.TelemetryMetrics.Reporter, + [ + metrics: Membrane.RTC.Engine.Endpoint.WebRTC.Metrics.metrics(), + name: JellyfishMetricsReporter + ]}, + {Jellyfish.MetricsScraper, scrape_interval}, + JellyfishWeb.Endpoint, + # Start the RoomService + Jellyfish.RoomService, + {Registry, keys: :unique, name: Jellyfish.RoomRegistry}, + {Registry, keys: :unique, name: Jellyfish.RequestHandlerRegistry}, + # Start the Telemetry supervisor (must be started after Jellyfish.RoomRegistry) + JellyfishWeb.Telemetry, + {Task.Supervisor, name: Jellyfish.TaskSupervisor} + ] ++ + if dist_config[:enabled] do + config_distribution(dist_config) + else + [] + end :ets.new(:rooms_to_tables, [:public, :set, :named_table]) - children = - if topologies == [] do - children - else - [{Cluster.Supervisor, [topologies, [name: Jellyfish.ClusterSupervisor]]} | children] - end - # See https://hexdocs.pm/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Jellyfish.Supervisor] @@ -51,4 +51,50 @@ defmodule Jellyfish.Application do JellyfishWeb.Endpoint.config_change(changed, removed) :ok end + + defp config_distribution(dist_config) do + ensure_epmd_started!() + + # Release always starts in a distributed mode + # so we have to start a node only in development. + # See env.sh.eex for more information. + unless Node.alive?() do + case Node.start(dist_config[:node_name]) do + {:ok, _} -> + :ok + + {:error, reason} -> + raise "Couldn't start Jellyfish node, reason: #{inspect(reason)}" + end + + Node.set_cookie(dist_config[:cookie]) + end + + topologies = [ + epmd_cluster: [ + strategy: Cluster.Strategy.Epmd, + config: [hosts: dist_config[:nodes]] + ] + ] + + [{Cluster.Supervisor, [topologies, [name: Jellyfish.ClusterSupervisor]]}] + end + + defp ensure_epmd_started!() do + case System.cmd("epmd", ["-daemon"]) do + {_, 0} -> + :ok + + _other -> + raise """ + Couldn't start epmd daemon. + Epmd is required to run Jellyfish in a distributed mode. + You can try to start it manually with: + + epmd -daemon + + and run Jellyfish again. + """ + end + end end diff --git a/lib/jellyfish/config_reader.ex b/lib/jellyfish/config_reader.ex index f90edeaf..507ae066 100644 --- a/lib/jellyfish/config_reader.ex +++ b/lib/jellyfish/config_reader.ex @@ -1,6 +1,8 @@ defmodule Jellyfish.ConfigReader do @moduledoc false + require Logger + def read_port_range(env) do if value = System.get_env(env) do with [str1, str2] <- String.split(value, "-"), @@ -48,19 +50,55 @@ defmodule Jellyfish.ConfigReader do end end - def read_nodes(env) do - value = System.get_env(env) - - if value not in ["", nil] do - value - |> String.split(" ", trim: true) - |> Enum.map(&String.to_atom(&1)) - end - end - def read_boolean(env) do if value = System.get_env(env) do String.downcase(value) not in ["false", "f", "0"] end end + + def read_dist_config() do + if read_boolean("JF_DIST_ENABLED") do + node_name_value = System.get_env("JF_NODE_NAME") + cookie_value = System.get_env("JF_COOKIE") + nodes_value = System.get_env("JF_NODES") || "" + + unset_var = + [{"JF_NODE_NAME", node_name_value}, {"JF_COOKIE", cookie_value}] + |> Enum.find(fn {_env_name, env_val} -> is_nil(env_val) end) + + if unset_var do + {unset_var_name, _} = unset_var + + raise """ + JF_DIST_ENABLED has been set but #{unset_var_name} remains unset. + JF_DIST_ENABLED requires following env vars to be also set: JF_NODE_NAME JF_COOKIE. + """ + end + + node_name = parse_node_name(node_name_value) + cookie = parse_cookie(cookie_value) + nodes = parse_nodes(nodes_value) + + if nodes == [] do + Logger.warning(""" + JF_DIST_ENABLED has been set but JF_NODES remains unset. + Make sure that at least one of your Jellyfish instances + has JF_NODES set. + """) + end + + [enabled: true, node_name: node_name, cookie: cookie, nodes: nodes] + else + [enabled: false, node_name: nil, cookie: nil, nodes: []] + end + end + + defp parse_node_name(node_name), do: String.to_atom(node_name) + defp parse_cookie(cookie_value), do: String.to_atom(cookie_value) + + defp parse_nodes(nodes_value) do + nodes_value + |> String.split(" ", trim: true) + |> Enum.map(&String.to_atom(&1)) + end end diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100644 index 00000000..aea4f448 --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,7 @@ +# release always starts in a distributed mode +# but to provide unified way of configuring Jellyfish +# distribution both in the local and production environment +# we introduce our own env vars and use them to +# set RELASE_NODE and RELASE_COOKIE +export RELEASE_NODE=${JF_NODE_NAME:-${RELEASE_NODE:-jellyfish}} +export RELEASE_COOKIE=${JF_COOKIE:-${RELEASE_COOKIE:-"$(cat "$RELEASE_ROOT/releases/COOKIE")"}} \ No newline at end of file diff --git a/test/jellyfish/config_reader_test.exs b/test/jellyfish/config_reader_test.exs index f80db9ec..d0b3169d 100644 --- a/test/jellyfish/config_reader_test.exs +++ b/test/jellyfish/config_reader_test.exs @@ -1,25 +1,32 @@ defmodule Jellyfish.ConfigReaderTest do - use ExUnit.Case, async: true + use ExUnit.Case + + # run this test synchronously as we use + # official env vars in read_dist_config test alias Jellyfish.ConfigReader defmacrop with_env(env, do: body) do - # get current env value, + # get current env(s) value(s), # execute test code, - # put back original env value + # put back original env(s) value(s) # # if env was not set, we have # to call System.delete_env as # System.put_env does not accept `nil` quote do - old = System.get_env(unquote(env)) + envs = List.wrap(unquote(env)) + old_envs = Enum.map(envs, fn env_name -> {env_name, System.get_env(env_name)} end) + unquote(body) - if old do - System.put_env(unquote(env), old) - else - System.delete_env(unquote(env)) - end + Enum.each(old_envs, fn {env_name, env_value} -> + if env_value do + System.put_env(env_name, env_value) + else + System.delete_env(env_name) + end + end) end end @@ -81,14 +88,36 @@ defmodule Jellyfish.ConfigReaderTest do end end - test "read_nodes/1" do - env_name = "JF_CONF_READER_TEST_NODES" + test "read_dist_config/0" do + with_env ["JF_DIST_ENABLED", "JF_COOKIE", "JF_NODE_NAME", "JF_NODES"] do + assert ConfigReader.read_dist_config() == [ + enabled: false, + node_name: nil, + cookie: nil, + nodes: [] + ] - with_env env_name do - System.put_env(env_name, "app1@127.0.0.1 app2@127.0.0.2") - assert ConfigReader.read_nodes(env_name) == [:"app1@127.0.0.1", :"app2@127.0.0.2"] - System.put_env(env_name, "") - assert ConfigReader.read_nodes(env_name) == nil + System.put_env("JF_DIST_ENABLED", "true") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_COOKIE", "testcookie") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_NODE_NAME", "testnodename@127.0.0.1") + + assert ConfigReader.read_dist_config() == [ + enabled: true, + node_name: :"testnodename@127.0.0.1", + cookie: :testcookie, + nodes: [] + ] + + System.put_env("JF_NODES", "testnodename1@127.0.0.1 testnodename2@127.0.0.1") + + assert ConfigReader.read_dist_config() == [ + enabled: true, + node_name: :"testnodename@127.0.0.1", + cookie: "testcookie", + nodes: [:"testnodename1@127.0.0.1", :"testnodename2@127.0.0.1"] + ] end end end