This repository has been archived by the owner on Jun 10, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
16 changed files
with
735 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
root = true | ||
|
||
[*] | ||
charset = utf-8 | ||
|
||
indent_style = tab | ||
indent_size = 4 | ||
|
||
end_of_line = lf | ||
insert_final_newline = true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
# This file is responsible for configuring your application | ||
# and its dependencies with the aid of the Mix.Config module. | ||
use Mix.Config | ||
|
||
# This configuration is loaded before any dependency and is restricted | ||
# to this project. If another project depends on this project, this | ||
# file won't be loaded nor affect the parent project. For this reason, | ||
# if you want to provide default values for your application for | ||
# 3rd-party users, it should be done in your "mix.exs" file. | ||
|
||
# You can configure for your application as: | ||
# | ||
# config :distributed, key: :value | ||
# | ||
# And access this configuration in your application as: | ||
# | ||
# Application.get_env(:distributed, :key) | ||
# | ||
# Or configure a 3rd-party app: | ||
# | ||
# config :logger, level: :info | ||
# | ||
|
||
# It is also possible to import configuration files, relative to this | ||
# directory. For example, you can emulate configuration per environment | ||
# by uncommenting the line below and defining dev.exs, test.exs and such. | ||
# Configuration from the imported file will override the ones defined | ||
# here (which is why it is important to import them last). | ||
# | ||
# import_config "#{Mix.env}.exs" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
defmodule Distributed do | ||
@moduledoc """ | ||
Make your systems distributed, replicated, scaled well, easily. | ||
[![Hex Version](https://img.shields.io/hexpm/v/distributed.svg?style=flat-square)](https://hex.pm/packages/distributed) [![Docs](https://img.shields.io/badge/api-docs-orange.svg?style=flat-square)](https://hexdocs.pm/distributed) [![Hex downloads](https://img.shields.io/hexpm/dt/distributed.svg?style=flat-square)](https://hex.pm/packages/distributed) [![GitHub](https://img.shields.io/badge/vcs-GitHub-blue.svg?style=flat-square)](https://github.com/ertgl/distributed) [![MIT License](https://img.shields.io/hexpm/l/distributed.svg?style=flat-square)](LICENSE.txt) | ||
--- | ||
### Tutorial | ||
This is an example of a replicated `GenServer`. | ||
defmodule Storage.KV do | ||
use GenServer | ||
def start_link() do | ||
GenServer.start_link(__MODULE__, [initial_state: %{}], name: __MODULE__.process_id()) | ||
end | ||
def init(opts \\\\ []) do | ||
{:ok, Keyword.get(opts, :initial_state, %{})} | ||
end | ||
def process_id() do | ||
Storage.KV | ||
end | ||
def handle_cast({:set, key, value}, state) do | ||
{:noreply, Map.put(state, key, value)} | ||
end | ||
def handle_call({:get, key, default}, _from, state) do | ||
{:reply, Map.get(state, key, default), state} | ||
end | ||
def handle_call({:has, key}, _from, state) do | ||
{:reply, Map.has_key?(state, key), state} | ||
end | ||
def handle_call({:pop, key, default}, _from, state) do | ||
{value, new_state} = Map.pop(state, key, default) | ||
{:reply, value, new_state} | ||
end | ||
def get(key, default \\\\ nil) do | ||
Distributed.Scaler.GenServer.call(__MODULE__.process_id(), {:get, key, default}) | ||
end | ||
def set(key, value) do | ||
Distributed.Replicator.GenServer.cast(__MODULE__.process_id(), {:set, key, value}) | ||
|> List.first() | ||
end | ||
def has?(key) do | ||
Distributed.Scaler.GenServer.call(__MODULE__.process_id(), {:has, key}) | ||
end | ||
def pop(key, default \\\\ nil) do | ||
Distributed.Replicator.GenServer.call(__MODULE__.process_id(), {:pop, key, default}) | ||
|> List.first() | ||
end | ||
end | ||
You can see the example as a small project on [GitHub](https://github.com/ertgl/storage). | ||
### Installation: | ||
If [you have Hex](https://hex.pm), the package can be installed | ||
by adding `:distributed` to your list of dependencies in `mix.exs`: | ||
def application do | ||
[ | ||
extra_applications: [ | ||
:distributed, | ||
], | ||
] | ||
end | ||
def deps do | ||
[ | ||
{:distributed, "~> 0.1.0"}, | ||
] | ||
end | ||
""" | ||
|
||
defmacro __using__(opts \\ []) do | ||
scaler_opts = Keyword.get(opts, :scaler, []) | ||
replicator_opts = Keyword.get(opts, :replicator, []) | ||
|
||
quote do | ||
use Distributed.Scaler, unquote(scaler_opts) | ||
use Distributed.Replicator, unquote(replicator_opts) | ||
end | ||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
defmodule Distributed.Application do | ||
# See http://elixir-lang.org/docs/stable/elixir/Application.html | ||
# for more information on OTP Applications | ||
@moduledoc false | ||
|
||
use Application | ||
|
||
def start(_type, _args) do | ||
import Supervisor.Spec, warn: false | ||
# Define workers and child supervisors to be supervised | ||
children = [ | ||
# Starts a worker by calling: Distributed.Worker.start_link(arg1, arg2, arg3) | ||
# worker(Distributed.Worker, [arg1, arg2, arg3]), | ||
worker(Distributed.Node, []), | ||
worker(Distributed.Node.Iterator, []), | ||
] | ||
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html | ||
# for other strategies and supported options | ||
opts = [ | ||
name: Distributed.Supervisor, | ||
strategy: :one_for_one, | ||
] | ||
Supervisor.start_link(children, opts) | ||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
defmodule Distributed.Node do | ||
@moduledoc """ | ||
Functions in `Distributed.Node` module help to know which node is the master and which are the slaves. | ||
""" | ||
|
||
use GenServer | ||
|
||
@doc false | ||
def start_link() do | ||
GenServer.start_link(__MODULE__, [], name: __MODULE__.process_id()) | ||
end | ||
|
||
@doc false | ||
def init(_opts \\ []) do | ||
{:ok, %{}} | ||
end | ||
|
||
@doc false | ||
def process_id() do | ||
{:global, Distributed.Node} | ||
end | ||
|
||
@doc false | ||
def handle_call(:list, _from, state) do | ||
{:reply, Node.list() ++ [Node.self()], state} | ||
end | ||
|
||
@doc false | ||
def handle_call(:master, _from, state) do | ||
{:reply, Node.self(), state} | ||
end | ||
|
||
@doc false | ||
def handle_call(:slaves, _from, state) do | ||
{:reply, Node.list(), state} | ||
end | ||
|
||
@doc """ | ||
Returns a list of names of all the nodes, including master. See also `Distributed.Node.Iterator`. | ||
""" | ||
@spec list(opts :: [any]) :: [atom] | ||
def list(opts \\ []) do | ||
timeout = Keyword.get(opts, :timeout, :infinity) | ||
GenServer.call(__MODULE__.process_id(), :list, timeout) | ||
end | ||
|
||
@doc """ | ||
Returns name of the master node. | ||
""" | ||
@spec master(opts :: [any]) :: atom | ||
def master(opts \\ []) do | ||
timeout = Keyword.get(opts, :timeout, :infinity) | ||
GenServer.call(__MODULE__.process_id(), :master, timeout) | ||
end | ||
|
||
@doc """ | ||
Returns a list of names of the slave nodes. | ||
""" | ||
@spec slaves(opts :: [any]) :: [atom] | ||
def slaves(opts \\ []) do | ||
timeout = Keyword.get(opts, :timeout, :infinity) | ||
GenServer.call(__MODULE__.process_id(), :slaves, timeout) | ||
end | ||
|
||
@doc """ | ||
Returns true if the `node` is the master node, otherwise false. | ||
""" | ||
@spec is_master?(node :: atom, opts :: [any]) :: boolean | ||
def is_master?(node, opts \\ []) do | ||
node == __MODULE__.master(opts) | ||
end | ||
|
||
@doc """ | ||
Returns true if the `node` is a slave node, otherwise false. | ||
""" | ||
@spec is_slave?(node :: atom, opts :: [any]) :: boolean | ||
def is_slave?(node, opts \\ []) do | ||
node in __MODULE__.slaves(opts) | ||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
defmodule Distributed.Node.Iterator do | ||
@moduledoc """ | ||
Functions in `Distributed.Node.Iterator` module help to iterate through all the nodes connected. | ||
""" | ||
|
||
use GenServer | ||
|
||
@doc false | ||
def start_link() do | ||
GenServer.start_link(__MODULE__, [], name: __MODULE__.process_id()) | ||
end | ||
|
||
@doc false | ||
def init(_opts \\ []) do | ||
{ | ||
:ok, | ||
%{ | ||
previous_nodes: [], | ||
}, | ||
} | ||
end | ||
|
||
@doc false | ||
def process_id() do | ||
{:global, __MODULE__} | ||
end | ||
|
||
@doc false | ||
def handle_call({:next, opts}, _from, %{previous_nodes: previous_nodes} = state) do | ||
skip_master = Keyword.get(opts, :skip_master, false) | ||
no_sweep = Keyword.get(opts, :no_sweep, false) | ||
nodes = case skip_master do | ||
true -> | ||
Distributed.Node.list() -- [Distributed.Node.master()] -- previous_nodes | ||
false -> | ||
Distributed.Node.list() -- previous_nodes | ||
end | ||
{need_to_repeat?, available_nodes} = case length(nodes) > 0 do | ||
true -> | ||
{false, nodes} | ||
false -> | ||
{true, Distributed.Node.list()} | ||
end | ||
next_node = List.first(available_nodes) | ||
timely_previous_nodes = case need_to_repeat? do | ||
false -> | ||
previous_nodes ++ [next_node] | ||
true -> | ||
[next_node] | ||
end | ||
case no_sweep do | ||
true -> | ||
{:reply, next_node, state} | ||
false -> | ||
{:reply, next_node, Map.put(state, :previous_nodes, timely_previous_nodes)} | ||
end | ||
end | ||
|
||
@doc """ | ||
Returns the next node's name in the iteration. When the iteration ends, it will repeat itself lazily. | ||
You can skip the master node via: | ||
skip_master: true | ||
Note: `skip_master` will not work if the master node is the only node in the network. | ||
You can also know which one the next node, without touching the order via: | ||
no_sweep: true | ||
""" | ||
@spec next(opts :: [any]) :: atom | ||
def next(opts \\ []) do | ||
timeout = Keyword.get(opts, :timeout, :infinity) | ||
GenServer.call(__MODULE__.process_id(), {:next, opts}, timeout) | ||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
defmodule Distributed.Parallel do | ||
@moduledoc """ | ||
The functions in `Distributed.Parallel` are not about scaling processes through all the nodes. They | ||
execute processes asynchronously on the node that uses the functions. | ||
If you want to execute processes through all nodes in parallel, please see `Distributed.Replication` module. | ||
""" | ||
|
||
@doc """ | ||
Returns the results of new processes started by the application of `fun` on CPU cores. | ||
See https://en.wikipedia.org/wiki/Map_(parallel_pattern) | ||
""" | ||
@spec map(enumerable :: Enum.enumerable, fun :: (() -> any), opts :: [any]) :: [any] | ||
def map(enumerable, fun, opts \\ []) | ||
when is_function(fun) | ||
do | ||
timeout = Keyword.get(opts, :timeout, :infinity) | ||
Enum.map(enumerable, &(Task.async(fn -> fun.(&1) end))) | ||
|> Enum.map(&(Task.await(&1, timeout))) | ||
end | ||
|
||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
defmodule Distributed.Replicator do | ||
|
||
defmacro __using__(opts \\ []) do | ||
node_opts = Keyword.get(opts, :node, []) | ||
gen_server_opts = Keyword.get(opts, :gen_server, []) | ||
|
||
quote do | ||
import(Distributed.Replicator.Node, unquote(node_opts)) | ||
import(Distributed.Replicator.GenServer, unquote(gen_server_opts)) | ||
end | ||
end | ||
|
||
end |
Oops, something went wrong.