From 5aa3209ec8f3af589e2b6d1818ad7ded362a7ce3 Mon Sep 17 00:00:00 2001 From: Renato Massaro Date: Wed, 8 Nov 2017 03:00:37 -0200 Subject: [PATCH] Add type specifications on TOP --- lib/hardware/internal/motherboard.ex | 15 +- lib/hardware/model/motherboard.ex | 15 ++ lib/process/action/process.ex | 39 ++-- lib/process/action/top.ex | 37 +++- lib/process/event/handler/top.ex | 18 +- lib/process/executable.ex | 3 + lib/process/internal/process.ex | 5 +- lib/process/model/process.ex | 183 +++++++++++++----- lib/process/model/process/resources.ex | 21 +- lib/process/model/processable.ex | 16 +- lib/process/model/top/allocator.ex | 37 +++- lib/process/model/top/scheduler.ex | 31 ++- lib/process/query/process.ex | 4 +- lib/process/query/top.ex | 15 +- lib/process/resources.ex | 50 ++++- lib/process/resources/behaviour.ex | 10 +- lib/process/resources/behaviour/default.ex | 52 ++++- lib/process/resources/behaviour/kv.ex | 68 ++++++- lib/process/resources/utils.ex | 5 + lib/server/model/server.ex | 3 + .../software_type/firewall/process_type.ex | 42 +--- lib/software/process/cracker/bruteforce.ex | 9 +- lib/software/process/cracker/overflow.ex | 9 +- lib/software/process/file/transfer.ex | 10 +- .../process/bank/account/password_reveal.ex | 7 +- lib/universe/bank/process/bank/transfer.ex | 9 +- test/entity/event/handler/database_test.exs | 27 ++- test/process/query/process_test.exs | 2 +- 28 files changed, 548 insertions(+), 194 deletions(-) diff --git a/lib/hardware/internal/motherboard.ex b/lib/hardware/internal/motherboard.ex index 92e17926..40378b2a 100644 --- a/lib/hardware/internal/motherboard.ex +++ b/lib/hardware/internal/motherboard.ex @@ -35,7 +35,7 @@ defmodule Helix.Hardware.Internal.Motherboard do end end - @spec fetch_by_nip(Network.id, NetworkConnection.ip) :: + @spec fetch_by_nip(Network.id, Network.ip) :: Motherboard.t | nil def fetch_by_nip(network_id, ip) do @@ -128,18 +128,7 @@ defmodule Helix.Hardware.Internal.Motherboard do end @spec resources(Motherboard.t) :: - %{ - cpu: non_neg_integer, - ram: non_neg_integer, - hdd: non_neg_integer, - net: %{ - Network.id => - %{ - uplink: non_neg_integer, - downlink: non_neg_integer - } - } - } + Motherboard.resources def resources(motherboard) do components_ids = get_components_ids(motherboard) diff --git a/lib/hardware/model/motherboard.ex b/lib/hardware/model/motherboard.ex index 107109c5..11d4eb93 100644 --- a/lib/hardware/model/motherboard.ex +++ b/lib/hardware/model/motherboard.ex @@ -5,6 +5,7 @@ defmodule Helix.Hardware.Model.Motherboard do import Ecto.Changeset alias Ecto.Changeset + alias Helix.Network.Model.Network alias Helix.Hardware.Model.Component alias Helix.Hardware.Model.ComponentSpec alias Helix.Hardware.Model.MotherboardSlot @@ -23,6 +24,20 @@ defmodule Helix.Hardware.Model.Motherboard do updated_at: NaiveDateTime.t } + @type resources :: + %{ + cpu: non_neg_integer, + ram: non_neg_integer, + hdd: non_neg_integer, + net: %{ + Network.id => + %{ + uplink: non_neg_integer, + downlink: non_neg_integer + } + } + } + @primary_key false schema "motherboards" do field :motherboard_id, Component.ID, diff --git a/lib/process/action/process.ex b/lib/process/action/process.ex index 1500179b..701a6d1c 100644 --- a/lib/process/action/process.ex +++ b/lib/process/action/process.ex @@ -1,5 +1,6 @@ defmodule Helix.Process.Action.Process do + alias Helix.Event alias Helix.Entity.Model.Entity alias Helix.Entity.Query.Entity, as: EntityQuery alias Helix.Network.Model.Connection @@ -25,16 +26,22 @@ defmodule Helix.Process.Action.Process do @type base_params :: %{ - :gateway_id => Server.idtb, - :target_id => Server.idtb, + :gateway_id => Server.id, + :target_id => Server.id, :data => Processable.t, - :type => String.t, - optional(:file_id) => File.idtb | nil, - optional(:network_id) => Network.idtb | nil, - optional(:connection_id) => Connection.idtb | nil, - optional(:objective) => map - } | term - + :type => Process.type, + :file_id => File.id | nil, + :network_id => Network.id | nil, + :connection_id => Connection.id | nil, + :objective => map, + :l_dynamic => Process.dynamic, + :r_dynamic => Process.dynamic, + :static => Process.static + } + + @spec create(base_params) :: + {:ok, Process.t, [ProcessCreatedEvent.t]} + | {:error, Process.changeset} def create(params) do with \ source_entity = EntityQuery.fetch_by_server(params.gateway_id), @@ -51,6 +58,8 @@ defmodule Helix.Process.Action.Process do end end + @spec delete(Process.t, Process.kill_reason) :: + {:ok, [ProcessCompletedEvent.t]} def delete(process = %Process{}, reason) do ProcessInternal.delete(process) @@ -73,6 +82,8 @@ defmodule Helix.Process.Action.Process do # {:ok, [event]} # end + @spec signal(Process.t, Process.signal, Process.signal_params) :: + {:ok, [Event.t]} def signal(process = %Process{}, signal, params \\ %{}) do {action, events} = signal_handler(signal, process, params) @@ -81,6 +92,8 @@ defmodule Helix.Process.Action.Process do {:ok, events ++ [signaled_event]} end + @spec signal_handler(Process.signal, Process.t, Process.signal_params) :: + {Processable.action, [Event.t]} defp signal_handler(:SIGTERM, process, _), do: Processable.complete(process.data, process) @@ -103,14 +116,16 @@ defmodule Helix.Process.Action.Process do # do: Processable.file_deleted(process.data, process, file) @spec prepare_create_params(base_params, Entity.id) :: - Process.create_params + Process.creation_params defp prepare_create_params(params, source_entity_id), do: Map.put(params, :source_entity_id, source_entity_id) @spec get_process_ips(base_params) :: {gateway_ip :: Network.ip, target_ip :: Network.ip} | {nil, nil} - defp get_process_ips(params = %{network_id: _}) do + defp get_process_ips(%{network_id: nil}), + do: {nil, nil} + defp get_process_ips(params) do gateway_ip = ServerQuery.get_ip(params.gateway_id, params.network_id) target_ip = @@ -122,6 +137,4 @@ defmodule Helix.Process.Action.Process do {gateway_ip, target_ip} end - defp get_process_ips(_), - do: {nil, nil} end diff --git a/lib/process/action/top.ex b/lib/process/action/top.ex index 610fb9c4..4671891b 100644 --- a/lib/process/action/top.ex +++ b/lib/process/action/top.ex @@ -5,7 +5,9 @@ defmodule Helix.Process.Action.TOP do alias Helix.Event alias Helix.Server.Model.Server alias Helix.Process.Action.Process, as: ProcessAction + alias Helix.Process.Internal.Process, as: ProcessInternal alias Helix.Process.Model.Process + alias Helix.Process.Model.Processable alias Helix.Process.Model.TOP alias Helix.Process.Query.Process, as: ProcessQuery alias Helix.Process.Query.TOP, as: TOPQuery @@ -13,16 +15,32 @@ defmodule Helix.Process.Action.TOP do alias Helix.Process.Event.TOP.BringMeToLife, as: TOPBringMeToLifeEvent alias Helix.Process.Event.TOP.Recalcado, as: TOPRecalcadoEvent + @type recalque_result :: + {:ok, [Process.t], [TOPRecalcadoEvent.t]} + | {:error, :resources} + + @typep recalque_opts :: term + + @spec complete(Process.t) :: + {:ok, [Event.t]} + | {:error, {:process, :running}} def complete(process) do case TOP.Scheduler.simulate(process) do {:completed, _process} -> ProcessAction.signal(process, :SIGTERM, %{reason: :completed}) {:running, _process} -> - {:error, {:process, :running}, []} + {:error, {:process, :running}} end end + @spec recalque(Server.id, recalque_opts) :: + recalque_result + @spec recalque(Process.t, recalque_opts) :: + %{ + gateway: recalque_result, + target: recalque_result + } def recalque(process_or_server, alloc_opts \\ []) def recalque(%Process{gateway_id: gateway_id, target_id: target_id}, opts) do @@ -34,6 +52,8 @@ defmodule Helix.Process.Action.TOP do def recalque(server_id = %Server.ID{}, opts), do: do_recalque(server_id, opts) + @spec do_recalque(Server.id, recalque_opts) :: + recalque_result defp do_recalque(server_id, alloc_opts) do resources = TOPQuery.load_top_resources(server_id) processes = ProcessQuery.get_processes_on_server(server_id) @@ -50,6 +70,8 @@ defmodule Helix.Process.Action.TOP do end end + @spec schedule(TOP.Allocator.allocation_successful) :: + [Process.t] defp schedule(%{allocated: processes, dropped: _dropped}) do # Organize all processes in two groups: the local ones and the remote ones # A local process was started on this very server, while a remote process @@ -119,6 +141,8 @@ defmodule Helix.Process.Action.TOP do processes end + @spec handle_forecast(TOP.Scheduler.forecast) :: + term docp """ `handle_forecast` aggregates the `Scheduler.forecast/1` result and guides it to the corresponding handlers. Check `handle_completed/1` and `handle_next/1` @@ -129,6 +153,8 @@ defmodule Helix.Process.Action.TOP do handle_next(next) end + @spec handle_completed([Process.t]) :: + term docp """ `handle_completed` receives processes that according to `Schedule.forecast/1` have already finished. We'll then complete each one and Emit their @@ -145,6 +171,8 @@ defmodule Helix.Process.Action.TOP do within our architecture, are not supposed to emit events. However, `handle_completed` happens within a spawned process, and as such the resulting events cannot be sent back to the original Handler/ActionFlow caller. + + Emits event. """ defp handle_completed([]), do: :noop @@ -156,6 +184,8 @@ defmodule Helix.Process.Action.TOP do end) end + @spec handle_next({Process.t, Process.time_left}) :: + term docp """ `handle_next` will receive the "next-to-be-completed" process, as defined by `Scheduler.forecast/1`. If a tuple is received, then we know there's a process @@ -163,6 +193,8 @@ defmodule Helix.Process.Action.TOP do Once the process is (supposedly) completed, TOP will receive the `TOPBringMeToLifeEvent`, which shall confirm the completion and actually complete the task. + + Emits TOPBringMeToLifeEvent.t after `time_left` seconds have elapsed. """ defp handle_next({process, time_left}) do wake_me_up = TOPBringMeToLifeEvent.new(process) @@ -174,7 +206,8 @@ defmodule Helix.Process.Action.TOP do defp handle_next(_), do: :noop - alias Helix.Process.Internal.Process, as: ProcessInternal + @spec handle_checkpoint([Process.t]) :: + term docp """ `handle_checkpoint` is responsible for handling the result of `Scheduler.checkpoint/1`, called during the `recalque` above. diff --git a/lib/process/event/handler/top.ex b/lib/process/event/handler/top.ex index 1c3b4170..d880f85b 100644 --- a/lib/process/event/handler/top.ex +++ b/lib/process/event/handler/top.ex @@ -20,8 +20,8 @@ defmodule Helix.Process.Event.Handler.TOP do Event.emit(events) # Can't wake up - {:error, {:process, :running}, []} -> - # This shouldn't happen... recalculate the TOP just in case + {:error, {:process, :running}} -> + # Weird but could happen. Recalculate the TOP just in case call_recalque(process) end end @@ -44,16 +44,17 @@ defmodule Helix.Process.Event.Handler.TOP do def recalque_handler(%_{confirmed: true}), do: :noop + @spec call_recalque(Process.t) :: + {gateway_recalque :: boolean, target_recalque :: boolean} defp call_recalque(process = %Process{}) do %{gateway: gateway_recalque, target: target_recalque} = TOPAction.recalque(process) gateway_recalque = case gateway_recalque do - {:ok, processes, events} -> + {:ok, _processes, events} -> Event.emit(events) - - {true, processes} + true _ -> false @@ -61,16 +62,15 @@ defmodule Helix.Process.Event.Handler.TOP do target_recalque = case target_recalque do - {:ok, processes, events} -> + {:ok, _processes, events} -> Event.emit(events) - - {true, processes} + true _ -> false end - {true, :todo} + {gateway_recalque, target_recalque} end def connection_closed(event = %ConnectionClosedEvent{}) do diff --git a/lib/process/executable.ex b/lib/process/executable.ex index 22e41a00..4e942c66 100644 --- a/lib/process/executable.ex +++ b/lib/process/executable.ex @@ -268,6 +268,9 @@ defmodule Helix.Process.Executable do {:error, %Ecto.Changeset{}} -> {:error, :internal} + + _ -> + {:error, :internal} end end end diff --git a/lib/process/internal/process.ex b/lib/process/internal/process.ex index bd2b3f4f..d76a1e00 100644 --- a/lib/process/internal/process.ex +++ b/lib/process/internal/process.ex @@ -5,6 +5,9 @@ defmodule Helix.Process.Internal.Process do alias Helix.Process.Model.Process alias Helix.Process.Repo + @spec create(Process.creation_params) :: + {:ok, Process.t} + | {:error, Process.changeset} def create(params) do params |> Process.create_changeset() @@ -29,7 +32,7 @@ defmodule Helix.Process.Internal.Process do |> Enum.map(&Process.format/1) end - @spec get_running_processes_of_type_on_server(Server.idt, String.t) :: + @spec get_running_processes_of_type_on_server(Server.idt, Process.type) :: [Process.t] def get_running_processes_of_type_on_server(gateway_id, type) do gateway_id diff --git a/lib/process/model/process.ex b/lib/process/model/process.ex index 7dae063b..dff95291 100644 --- a/lib/process/model/process.ex +++ b/lib/process/model/process.ex @@ -14,6 +14,7 @@ defmodule Helix.Process.Model.Process do import Ecto.Changeset + alias Ecto.Changeset alias HELL.Constant alias HELL.MapUtils alias HELL.NaiveStruct @@ -26,57 +27,78 @@ defmodule Helix.Process.Model.Process do alias Helix.Process.Model.TOP alias __MODULE__, as: Process - @type type :: String.t - - @type t :: term - # @type t :: %__MODULE__{ - # process_id: id, - # gateway_id: Server.id, - # source_entity_id: Entity.id, - # target_id: Server.id, - # file_id: File.id | nil, - # network_id: Network.id | nil, - # connection_id: Connection.id | nil, - # process_data: Processable.t, - # process_type: type, - # state: State.state, - # limitations: Limitations.t, - # objective: Resources.t, - # processed: Resources.t, - # allocated: Resources.t, - # priority: 0..5, - # minimum: map, - # creation_time: DateTime.t, - # updated_time: DateTime.t, - # estimated_time: DateTime.t | nil - # } - - # @type process :: %__MODULE__{} + @type t :: + %__MODULE__{ + process_id: id, + gateway_id: Server.id, + source_entity_id: Entity.id, + target_id: Server.id, + file_id: File.id | nil, + connection_id: Connection.id | nil, + network_id: Network.id | nil, + data: term, + type: type, + priority: term, + l_allocated: Process.Resources.t | nil, + r_allocated: Process.Resources.t | nil, + r_limit: limit, + l_limit: limit, + l_reserved: Process.Resources.t, + r_reserved: Process.Resources.t, + last_checkpoint_time: DateTime.t, + static: static, + l_dynamic: dynamic, + r_dynamic: dynamic, + local?: boolean | nil, + next_allocation: Process.Resources.t | nil, + state: state | nil, + creation_time: DateTime.t, + time_left: non_neg_integer | nil, + completion_date: DateTime.t | nil + } - @type creation_params :: %{ - # :gateway_id => Server.idtb, - # :source_entity_id => Entity.idtb, - # :target_id => Server.idtb, - # :process_data => Processable.t, - # :process_type => String.t, - # optional(:file_id) => File.idtb, - # optional(:network_id) => Network.idtb, - # optional(:connection_id) => Connection.idtb, - # optional(:objective) => map - } + @typep limit :: Process.Resources.t | %{} + + @type type :: + :file_upload + | :file_download + | :cracker_bruteforce + | :cracker_overflow + + @type signal :: + :SIGTERM + | :SIGKILL + | :SIGSTOP + | :SIGCONT + | :SIGPRIO + | :SIGCONND + | :SIGFILED - @type update_params :: %{ - # optional(:state) => State.state, - # optional(:priority) => 0..5, - # optional(:creation_time) => DateTime.t, - # optional(:updated_time) => DateTime.t, - # optional(:estimated_time) => DateTime.t | nil, - # optional(:limitations) => map, - # optional(:objective) => map, - # optional(:processed) => map, - # optional(:allocated) => map, - # optional(:minimum) => map, - # optional(:process_data) => Processable.t + @type signal_params :: + %{reason: kill_reason} + | %{priority: term} + | %{connection: Connection.t} + | %{file: File.t} + + @type kill_reason :: + :completed + | :killed + + @type changeset :: %Changeset{data: %__MODULE__{}} + + @type creation_params :: %{ + :gateway_id => Server.id, + :source_entity_id => Entity.id, + :target_id => Server.id, + :data => Processable.t, + :type => type, + :network_id => Network.id | nil, + :file_id => File.id | nil, + :connection_id => Connection.id | nil, + :objective => map, + :l_dynamic => dynamic, + :r_dynamic => dynamic, + :static => static, } @creation_fields [ @@ -109,6 +131,35 @@ defmodule Helix.Process.Model.Process do :priority ] + @type state :: + :waiting_allocation + | :running + | :paused + + @type time_left :: float + + @type resource :: + :cpu + | :ram + | :dlk + | :ulk + + @type dynamic :: [resource] + + @type static :: + %{ + paused: static_usage, + running: static_usage + } + + @typep static_usage :: + %{ + cpu: non_neg_integer, + ram: non_neg_integer, + dlk: non_neg_integer, + ulk: non_neg_integer + } + # Similar to `task_struct` on `sched.h` ;-) @primary_key false schema "processes" do @@ -214,6 +265,8 @@ defmodule Helix.Process.Model.Process do virtual: true end + @spec create_changeset(creation_params) :: + changeset def create_changeset(params) do %__MODULE__{} |> cast(params, @creation_fields) @@ -221,6 +274,8 @@ defmodule Helix.Process.Model.Process do |> put_defaults() end + @spec format(raw_process :: t) :: + t @doc """ Converts the retrieved process from the Database into TOP's internal format. Notably, it: @@ -243,21 +298,29 @@ defmodule Helix.Process.Model.Process do |> estimate_duration() end + @spec get_dynamic(t) :: + [resource] def get_dynamic(%{local?: true, l_dynamic: dynamic}), do: dynamic def get_dynamic(%{local?: false, r_dynamic: dynamic}), do: dynamic + @spec get_limit(t) :: + limit def get_limit(%{local?: true, l_limit: limit}), do: limit def get_limit(%{local?: false, r_limit: limit}), do: limit + @spec get_last_update(t) :: + DateTime.t def get_last_update(p = %{last_checkpoint_time: nil}), do: p.creation_time def get_last_update(%{last_checkpoint_time: last_checkpoint_time}), do: last_checkpoint_time + @spec infer_usage(t) :: + t def infer_usage(process) do l_alloc = Process.Resources.min(process.l_limit, process.l_reserved) r_alloc = Process.Resources.min(process.r_limit, process.r_reserved) @@ -288,6 +351,8 @@ defmodule Helix.Process.Model.Process do } end + @spec estimate_duration(t) :: + t defp estimate_duration(process = %Process{}) do {_, time_left} = TOP.Scheduler.estimate_completion(process) @@ -313,6 +378,8 @@ defmodule Helix.Process.Model.Process do |> Map.replace(:time_left, time_left) end + @spec format_resources(t) :: + t defp format_resources(process = %Process{}) do process |> format_objective() @@ -322,20 +389,28 @@ defmodule Helix.Process.Model.Process do |> format_reserved() end + @spec format_objective(t) :: + t defp format_objective(p = %{objective: objective}), do: %{p| objective: Process.Resources.format(objective)} + @spec format_processed(t) :: + t defp format_processed(p = %{processed: nil}), do: p defp format_processed(p = %{processed: processed}), do: %{p| processed: Process.Resources.format(processed)} + @spec format_static(t) :: + t defp format_static(p = %{static: static}) do static = MapUtils.atomize_keys(static) %{p| static: static} end + @spec format_limits(t) :: + t defp format_limits(p) do l_limit = p.l_limit @@ -353,6 +428,8 @@ defmodule Helix.Process.Model.Process do } end + @spec format_reserved(t) :: + t defp format_reserved(p) do %{p| l_reserved: Process.Resources.format(p.l_reserved), @@ -360,6 +437,8 @@ defmodule Helix.Process.Model.Process do } end + @spec get_state(t) :: + state defp get_state(%{priority: 0}), do: :paused defp get_state(process) do @@ -370,6 +449,8 @@ defmodule Helix.Process.Model.Process do end end + @spec put_defaults(changeset) :: + changeset defp put_defaults(changeset) do changeset |> put_change(:creation_time, DateTime.utc_now()) @@ -391,7 +472,7 @@ defmodule Helix.Process.Model.Process do def by_id(query \\ Process, id), do: where(query, [p], p.process_id == ^id) - @spec from_type_list(Queryable.t, [String.t]) :: + @spec from_type_list(Queryable.t, [Process.type]) :: Queryable.t def from_type_list(query \\ Process, type_list), do: where(query, [p], p.type in ^type_list) @@ -429,7 +510,7 @@ defmodule Helix.Process.Model.Process do def by_connection(query \\ Process, id), do: where(query, [p], p.connection_id == ^id) - @spec by_type(Queryable.t, String.t) :: + @spec by_type(Queryable.t, Process.type) :: Queryable.t def by_type(query \\ Process, type), do: where(query, [p], p.type == ^type) diff --git a/lib/process/model/process/resources.ex b/lib/process/model/process/resources.ex index 4e1683f4..7d1e81e4 100644 --- a/lib/process/model/process/resources.ex +++ b/lib/process/model/process/resources.ex @@ -2,6 +2,8 @@ defmodule Helix.Process.Model.Process.Resources.Utils do alias Helix.Network.Model.Network + @spec format_network(Network.idtb, term) :: + {Network.id, term} def format_network(key = %Network.ID{}, value), do: {key, value} def format_network(key, value), @@ -12,8 +14,25 @@ import Helix.Process.Resources resources Helix.Process.Model.Process.Resources do - alias Helix.Process.Resources.Behaviour + alias Helix.Network.Model.Network alias Helix.Process.Model.Process.Resources.Utils, as: ResourcesUtils + alias Helix.Process.Resources.Behaviour + + @type t :: + %{ + ram: number, + cpu: number, + dlk: %{Network.id => number}, + ulk: %{Network.id => number} + } + + @type map_t(type) :: + %{ + ram: type, + cpu: type, + dlk: %{Network.id => type}, + ulk: %{Network.id => type} + } resource RAM, behaviour: Behaviour.Default diff --git a/lib/process/model/processable.ex b/lib/process/model/processable.ex index 0f1ca1cc..72be1a20 100644 --- a/lib/process/model/processable.ex +++ b/lib/process/model/processable.ex @@ -1,9 +1,11 @@ defprotocol Helix.Process.Model.Processable do - # alias Helix.Process.Model.Process - # @type resource :: :cpu | :ram | :dlk | :ulk + alias Helix.Event + alias Helix.Network.Model.Connection + alias Helix.Process.Model.Process + @type action :: :delete | :pause @@ -11,14 +13,16 @@ defprotocol Helix.Process.Model.Processable do | :renice | :restart - # @spec complete(t, Process.t | Ecto.Changeset.t) :: - # {[Process.t | Ecto.Changeset.t] | Process.t | Ecto.Changeset.t, [struct]} + @spec complete(t, Process.t) :: + {action, [Event.t]} def complete(data, process) - # @spec kill(t, Process.t | Ecto.Changeset.t, atom) :: - # {[Process.t | Ecto.Changeset.t] | Process.t | Ecto.Changeset.t, [struct]} + @spec kill(t, Process.t, Process.kill_reason) :: + {action, [Event.t]} def kill(data, process, reason) + @spec connection_closed(t, Process.t, Connection.t) :: + {action, [Event.t]} def connection_closed(data, process, connection) @spec after_read_hook(term) :: diff --git a/lib/process/model/top/allocator.ex b/lib/process/model/top/allocator.ex index f8d0bee7..eea6549e 100644 --- a/lib/process/model/top/allocator.ex +++ b/lib/process/model/top/allocator.ex @@ -1,7 +1,12 @@ defmodule Helix.Process.Model.TOP.Allocator do + alias Helix.Server.Model.Server alias Helix.Process.Model.Process + @type shares :: number + + @spec identify_origin(Server.id, [Process.t]) :: + [Process.t] defp identify_origin(server_id, processes) do Enum.map(processes, fn process -> local? = @@ -15,8 +20,23 @@ defmodule Helix.Process.Model.TOP.Allocator do end) end - # @spec allocate(server_resources, [Process.t]) :: - # [{Process.t, allocated_resources}] + @type allocated_process :: {Process.t | term, Process.Resources.t | term} + + @type allocation_successful :: + %{ + dropped: [Process.t], + allocated: [Process.t] + } + + @type allocation_failed :: + {:error, :resources, [Process.t]} + + @type allocation_result :: + {:ok, allocation_successful} + | allocation_failed + + @spec allocate(Server.id, Process.Resources.t, [Process.t], opts :: term) :: + allocation_result def allocate(server_id, total_resources, processes, _opts \\ []) do # forced_allocation? = opts[:force] || false @@ -77,12 +97,17 @@ defmodule Helix.Process.Model.TOP.Allocator do end end + @spec merge_allocation([allocated_process]) :: + [Process.t] defp merge_allocation(allocated_processes) do Enum.map(allocated_processes, fn {process, new_alloc} -> %{process| next_allocation: new_alloc} end) end + @spec overflow?(Process.Resources.t, [allocated_process]) :: + {true, heaviest :: [Process.t]} + | false defp overflow?(remaining_resources, allocated_processes) do # Checks whether any of the resources are in overflow (usage > available) overflow? = @@ -112,6 +137,8 @@ defmodule Helix.Process.Model.TOP.Allocator do end end + @spec static_allocation([Process.t]) :: + {allocated :: Process.Resources.t, [allocated_process]} def static_allocation(processes) do initial = Process.Resources.initial() @@ -130,7 +157,8 @@ defmodule Helix.Process.Model.TOP.Allocator do end) end - # @type share :: %{cpu: ....} + @spec dynamic_allocation(Process.Resources.t, [allocated_process]) :: + {Process.Resources.t, [allocated_process]} def dynamic_allocation(available_resources, allocated_processes) do initial = Process.Resources.initial() @@ -180,9 +208,10 @@ defmodule Helix.Process.Model.TOP.Allocator do {total_alloc, acc ++ [{process, proc_allocation}]} end) - end + @spec remaining_allocation(Process.Resources.t, [allocated_process]) :: + {Process.Resources.t, [allocated_process]} def remaining_allocation(available_resources, allocated_processes) do # Exclude processes that have limits # Note that this is wrong: it's possible that a process with limits would diff --git a/lib/process/model/top/scheduler.ex b/lib/process/model/top/scheduler.ex index 52b24d6e..e264c38e 100644 --- a/lib/process/model/top/scheduler.ex +++ b/lib/process/model/top/scheduler.ex @@ -3,6 +3,18 @@ defmodule Helix.Process.Model.TOP.Scheduler do alias Ecto.Changeset alias Helix.Process.Model.Process + @type forecast :: + %{ + next: {Process.t, Process.time_left} | nil, + paused: [Process.t], + completed: [Process.t], + running: [Process.t] + } + + @spec simulate(Process.t) :: + {:completed, Process.t} + | {:running, Process.t} + | {:paused, Process.t} def simulate(process = %{state: :paused}), do: {:paused, process} def simulate(process = %{state: :waiting_allocation}) do @@ -50,6 +62,8 @@ defmodule Helix.Process.Model.TOP.Scheduler do end end + @spec forecast([Process.t]) :: + forecast def forecast(processes) do initial_acc = %{next: nil, paused: [], completed: [], running: []} @@ -76,18 +90,23 @@ defmodule Helix.Process.Model.TOP.Scheduler do seconds -> %{acc| running: acc.running ++ [process], - next: sort_next_completion(acc, {process, seconds}) + next: sort_next_completion(acc.next, {process, seconds}) } end end) end + @spec estimate_completion(Process.t) :: + {Process.t, Process.time_left | -1 | :infinity} def estimate_completion(process) do process |> simulate() |> seconds_for_completion() end + @spec checkpoint(Process.t) :: + {true, Process.changeset} + | false def checkpoint(%{l_reserved: alloc, next_allocation: alloc, local?: true}), do: false def checkpoint(%{r_reserved: alloc, next_allocation: alloc, local?: false}), @@ -121,6 +140,8 @@ defmodule Helix.Process.Model.TOP.Scheduler do {true, changeset} end + @spec get_simulation_duration(Process.t) :: + pos_integer defp get_simulation_duration(process) do now = DateTime.utc_now() last_update = Process.get_last_update(process) @@ -128,6 +149,8 @@ defmodule Helix.Process.Model.TOP.Scheduler do DateTime.diff(now, last_update, :millisecond) end + @spec seconds_for_completion({:paused | :completed | :running, Process.t}) :: + {Process.t, Process.time_left | -1 | :infinity} defp seconds_for_completion({:paused, process}), do: {process, :infinity} defp seconds_for_completion({:completed, process}), @@ -157,9 +180,11 @@ defmodule Helix.Process.Model.TOP.Scheduler do {process, estimated_seconds} end - defp sort_next_completion(%{next: nil}, {process, seconds}), + @spec sort_next_completion(nil | {Process.t, term}, {Process.t, term}) :: + {Process.t, term} + defp sort_next_completion(nil, {process, seconds}), do: {process, seconds} - defp sort_next_completion(%{next: current}, candidate) do + defp sort_next_completion(current, candidate) do {_, cur_seconds} = current {_, candidate_seconds} = candidate diff --git a/lib/process/query/process.ex b/lib/process/query/process.ex index a83a6a6f..57f33a83 100644 --- a/lib/process/query/process.ex +++ b/lib/process/query/process.ex @@ -24,7 +24,7 @@ defmodule Helix.Process.Query.Process do defdelegate fetch(id), to: ProcessInternal - @spec get_running_processes_of_type_on_server(Server.idt, String.t) :: + @spec get_running_processes_of_type_on_server(Server.idt, Process.type) :: [Process.t] @doc """ Fetches processes running on `gateway` that are of `type` @@ -85,7 +85,7 @@ defmodule Helix.Process.Query.Process do The generated code is something like: ``` - def get_custom(type = "process_type", server_id, %{file_id: file_id}) do + def get_custom(type = :process_type, server_id, %{file_id: file_id}) do server_id |> get_running_processes_of_type_on_server(type) |> Enum.fiter(&(&1.file_id == file_id)) diff --git a/lib/process/query/top.ex b/lib/process/query/top.ex index 20a0a837..cbd6a533 100644 --- a/lib/process/query/top.ex +++ b/lib/process/query/top.ex @@ -3,13 +3,10 @@ defmodule Helix.Process.Query.TOP do alias Helix.Hardware.Query.Motherboard, as: MotherboardQuery alias Helix.Server.Model.Server alias Helix.Server.Query.Server, as: ServerQuery + alias Helix.Process.Model.Process - def load_top_resources(server_id = %Server.ID{}) do - server_id - |> ServerQuery.fetch() - |> load_top_resources() - end - + @spec load_top_resources(Server.idt) :: + Process.Resources.t def load_top_resources(server = %Server{}) do resources = server.motherboard_id @@ -42,4 +39,10 @@ defmodule Helix.Process.Query.TOP do ulk: server_ulk } end + + def load_top_resources(server_id = %Server.ID{}) do + server_id + |> ServerQuery.fetch() + |> load_top_resources() + end end diff --git a/lib/process/resources.ex b/lib/process/resources.ex index 9a99d68a..8ce9f1d8 100644 --- a/lib/process/resources.ex +++ b/lib/process/resources.ex @@ -22,6 +22,9 @@ defmodule Helix.Process.Resources do defmacro __before_compile__(_) do quote do + alias Helix.Process.Model.Process + alias Helix.Process.Model.TOP + # Maps the resource (name) to its module. @res_modules ( Enum.reduce(@resources, %{}, fn resource, acc -> @@ -32,12 +35,23 @@ defmodule Helix.Process.Resources do end) ) + @spec map(t, function) :: + map_t(term) def map(res_a, fun), do: dispatch(:map, res_a, [fun]) + @spec reduce(t, term, function) :: + term + def reduce(resource, initial, function), + do: dispatch(:reduce, resource, [initial, function]) + + @spec initial :: + t def initial, do: dispatch_create :initial + @spec format(t) :: + t def format(resources) do # Make sure our keys are valid, and all keys are defined on the resource # (If any key is missing, it will be populated to its initial value) @@ -49,6 +63,8 @@ defmodule Helix.Process.Resources do dispatch(:format, resources) end + @spec reject_empty(t) :: + t | %{} def reject_empty(resources) do Enum.reject(resources, fn {res, val} -> val == call_resource(res, :initial, []) @@ -56,6 +72,8 @@ defmodule Helix.Process.Resources do |> Map.new() end + @spec prepare(t) :: + t def prepare(resources) do # First and foremost, we must ensure that all keys have been transformed # into atoms. If they came from the DB, they will be a string. @@ -78,36 +96,53 @@ defmodule Helix.Process.Resources do end) end - def reduce(resource, initial, function), - do: dispatch(:reduce, resource, [initial, function]) - + @spec sum(t, t) :: + t def sum(res_a, res_b), do: dispatch_merge(:sum, res_a, res_b) + @spec sub(t, t) :: + t def sub(res_a, res_b), do: dispatch_merge(:sub, res_a, res_b) + @spec mul(t, t) :: + t def mul(res_a, res_b), do: dispatch_merge(:mul, res_a, res_b) + @spec div(t, t) :: + t def div(res_a, res_b), do: dispatch_merge(:div, res_a, res_b) + @spec get_shares(Process.t) :: + t def get_shares(process), do: dispatch_create :get_shares, [process] + @spec resource_per_share(t, t) :: + t def resource_per_share(resources, shares), do: dispatch_merge :resource_per_share, resources, shares + @spec allocate_static(Process.t) :: + t def allocate_static(process), do: dispatch_create :allocate_static, [process] + @spec allocate_dynamic(t, t, Process.t) :: + t def allocate_dynamic(shares, res_per_share, process), do: dispatch_merge :allocate_dynamic, shares, res_per_share, [process] + @spec allocate(t, t) :: + t def allocate(dynamic_alloc, static_alloc), do: dispatch_merge :allocate, dynamic_alloc, static_alloc + @spec completed?(t, t) :: + boolean def completed?(processed, objective) do :completed? |> dispatch_merge(processed, objective) @@ -115,9 +150,13 @@ defmodule Helix.Process.Resources do |> Enum.all?(fn {_res, status} -> status == true end) end + @spec overflow?(t, [TOP.Allocator.allocated_process]) :: + map_t({true, Process.t} | false) def overflow?(resources, processes), do: dispatch(:overflow?, resources, [processes]) + @spec mirror(t) :: + map_t(Process.resource) def mirror(resources) do Enum.reduce(resources, %{}, fn {res, val}, acc -> mirror_res = call_resource(res, :mirror, []) @@ -128,6 +167,8 @@ defmodule Helix.Process.Resources do end) end + @spec max(t) :: + number def max(resources) do resources |> reduce(0, fn acc, v -> max(acc, v) end) @@ -140,6 +181,8 @@ defmodule Helix.Process.Resources do |> elem(1) end + @spec min(t, t) :: + t | %{} def min(res1, res2) do :op_map |> dispatch_merge(res1, res2, [&min/2]) @@ -281,6 +324,7 @@ defmodule Helix.Process.Resources do quote location: :keep do + def unquote(op)(unquote(a), unquote(b)) do unquote(block) |> build() diff --git a/lib/process/resources/behaviour.ex b/lib/process/resources/behaviour.ex index 9c64c19c..2e54d5a9 100644 --- a/lib/process/resources/behaviour.ex +++ b/lib/process/resources/behaviour.ex @@ -10,13 +10,13 @@ defmodule Helix.Process.Resources.Behaviour do @callback build(term) :: resource @callback initial() :: resource - # # Operations + # Operations @callback sum(resource, resource) :: resource @callback sub(resource, resource) :: resource @callback mul(resource, resource) :: resource @callback div(resource, resource) :: resource - # # Allocation logic + # Allocation logic @callback get_shares(process) :: shares :: resource @callback allocate_dynamic( shares :: resource, @@ -28,7 +28,7 @@ defmodule Helix.Process.Resources.Behaviour do @callback allocate(dynamic :: resource, static :: resource) :: resource # Flow checks / verifications - @callback overflow?(resource, {process, allocations :: resource}) :: - false - | {true, heaviest :: process} + @callback overflow?(resource, [{process, allocations :: resource}]) :: + {true, heaviest :: process} + | false end diff --git a/lib/process/resources/behaviour/default.ex b/lib/process/resources/behaviour/default.ex index 4b6e5535..d342647b 100644 --- a/lib/process/resources/behaviour/default.ex +++ b/lib/process/resources/behaviour/default.ex @@ -6,6 +6,7 @@ defmodule Helix.Process.Resources.Behaviour.Default do quote location: :keep do alias Helix.Process.Model.Process + alias Helix.Process.Model.TOP alias Helix.Process.Resources.Utils, as: ResourceUtils @behaviour Helix.Process.Resources.Behaviour @@ -14,51 +15,78 @@ defmodule Helix.Process.Resources.Behaviour.Default do @formatter unquote(args)[:formatter] || &__MODULE__.default_formatter/1 @mirror unquote(args)[:mirror] || @name + @type t :: number + @type initial :: t + # Generic data manipulation + @spec reduce(t, term, function) :: + term def reduce(resource, initial, function), do: function.(initial, resource) + @spec map(t, function) :: + term def map(resource, function), do: function.(resource) + @spec op_map(t, t, function) :: + t def op_map(a, b, function), do: function.(a, b) # Creation & formatting of resource + @spec build(number) :: + t def build(value), do: value |> ResourceUtils.ensure_float() + @spec initial :: + initial def initial, do: build(0) + @spec format(t) :: + t def format(resource), do: @formatter.(resource) + @spec default_formatter(t) :: + t def default_formatter(v), do: v # Basic operations + @spec sum(t, t) :: + t sum(a, b) do a + b end + @spec sub(t, t) :: + t sub(a, b) do a - b end + @spec div(t, t) :: + t div(a, b) do ResourceUtils.safe_div(a, b, &initial/0) end + @spec mul(t, t) :: + t mul(a, b) do a * b end # Allocation logic + @spec get_shares(Process.t) :: + t get_shares(process = %{priority: priority}) do dynamic_res = Process.get_dynamic(process) @@ -73,21 +101,28 @@ defmodule Helix.Process.Resources.Behaviour.Default do end end - def mirror do - @mirror - end + @spec mirror :: + Process.resource + def mirror, + do: @mirror + @spec can_allocate?(Process.t) :: + boolean defp can_allocate?(%{processed: nil}), do: true defp can_allocate?(%{processed: processed, objective: objective}), do: Map.fetch!(objective, @name) >= Map.get(processed, @name, 0) + @spec resource_per_share(t, t) :: + t resource_per_share(resources, shares) do res_per_share = __MODULE__.div(resources, shares) res_per_share >= 0 && res_per_share || 0.0 end + @spec allocate_static(Process.t) :: + t allocate_static(%{local?: false}) do initial() end @@ -105,6 +140,8 @@ defmodule Helix.Process.Resources.Behaviour.Default do |> Map.get(@name, initial()) end + @spec allocate_dynamic(t, t, Process.t) :: + t allocate_dynamic(shares, res_per_share, process) do dynamic = Process.get_dynamic(process) @@ -115,13 +152,20 @@ defmodule Helix.Process.Resources.Behaviour.Default do end end + @spec allocate(t, t) :: + t allocate(dynamic_alloc, static_alloc) do sum(dynamic_alloc, static_alloc) end + @spec completed?(t, t) :: + boolean def completed?(processed, objective), do: processed >= objective + @spec overflow?(t, [TOP.Allocator.allocated_process]) :: + {true, heaviest :: Process.t} + | false def overflow?(res, allocated_processes) do # Due to rounding errors, we may have a "valid overflow" of a few units if res < -1 do @@ -131,6 +175,8 @@ defmodule Helix.Process.Resources.Behaviour.Default do end end + @spec find_heaviest([TOP.Allocator.allocated_process]) :: + Process.t defp find_heaviest(allocated_processes) do allocated_processes |> Enum.sort_by(fn {process, resources} -> diff --git a/lib/process/resources/behaviour/kv.ex b/lib/process/resources/behaviour/kv.ex index f521b1df..1ab0b872 100644 --- a/lib/process/resources/behaviour/kv.ex +++ b/lib/process/resources/behaviour/kv.ex @@ -8,6 +8,7 @@ defmodule Helix.Process.Resources.Behaviour.KV do formatter = unquote(args)[:formatter] alias Helix.Process.Model.Process + alias Helix.Process.Model.TOP alias Helix.Process.Resources.Utils, as: ResourceUtils @behaviour Helix.Process.Resources.Behaviour @@ -17,8 +18,18 @@ defmodule Helix.Process.Resources.Behaviour.KV do @formatter unquote(args)[:formatter] || &__MODULE__.default_formatter/2 @mirror unquote(args)[:mirror] || @name + @type key :: term + @type value :: number + + @type t :: %{key => value} | %{} + @type initial :: %{} + + @type map_t(type) :: %{key => type} | %{} + # Generic data manipulation + @spec map(t, function) :: + map_t(term) def map(resource, function) do Enum.reduce(resource, %{}, fn {key, value}, acc -> new_value = function.(value) @@ -29,12 +40,16 @@ defmodule Helix.Process.Resources.Behaviour.KV do end) end + @spec reduce(t, term, function) :: + term def reduce(resource, initial, function) do Enum.reduce(resource, initial, fn {key, value}, acc -> function.(acc, value) end) end + @spec op_map(t, t, function) :: + t def op_map(a, b, fun) do keys = get_keys(a, b) @@ -45,8 +60,8 @@ defmodule Helix.Process.Resources.Behaviour.KV do # Creation & formatting of resource - def build([%{}]), - do: %{} + @spec build(t | [t]) :: + t def build(entries) do Enum.reduce(entries, %{}, fn {key, value}, acc -> @@ -56,9 +71,13 @@ defmodule Helix.Process.Resources.Behaviour.KV do end) end + @spec initial :: + initial def initial, do: build([]) + @spec format(map_t(term)) :: + t def format(resource) do Enum.reduce(resource, %{}, fn {key, value}, acc -> {k, v} = @formatter.(key, value) @@ -69,15 +88,21 @@ defmodule Helix.Process.Resources.Behaviour.KV do end) end + @spec default_formatter(term, term) :: + {key, number} def default_formatter(k, v), do: {k, v} # Basic operations + @spec sum(t, t) :: + t sum(a, b) do op_map(a, b, &Kernel.+/2) end + @spec sub(t, t) :: + t sub(a, b) do # Ensure missing elements (exist on `b` but not on `a`) are filled as 0. # a = fill_missing(a, b) @@ -85,22 +110,30 @@ defmodule Helix.Process.Resources.Behaviour.KV do op_map(a, b, &Kernel.-/2) end + @spec mul(t, t) :: + t mul(a, b) do op_map(a, b, &Kernel.*/2) end + @spec div(t, t) :: + t div(a, b) do op_map(a, b, fn a, b -> ResourceUtils.safe_div(a, b, &initial/0) end) end # Allocation logic + @spec fill_missing(t, t, value) :: + t defp fill_missing(a, b, value \\ 0) do Enum.reduce(get_keys(a, b), a, fn key, acc -> Map.put_new(acc, key, value) end) end + @spec get_shares(Process.t) :: + t get_shares(process = %{priority: priority}) do dynamic = Process.get_dynamic(process) @@ -117,10 +150,13 @@ defmodule Helix.Process.Resources.Behaviour.KV do end end - def mirror do - @mirror - end + @spec mirror :: + Process.resource + def mirror, + do: @mirror + @spec can_allocate?(Process.t, key) :: + boolean defp can_allocate?(%{processed: nil}, _), do: true defp can_allocate?(%{local?: false}, _), @@ -136,6 +172,8 @@ defmodule Helix.Process.Resources.Behaviour.KV do value_objective > value_processed end + @spec resource_per_share(t, t) :: + t resource_per_share(resources, shares) do # If there are fields defined on `resources` which are not on `shares`, # then we must "fill" `shares` with zero, since this means that the @@ -147,12 +185,15 @@ defmodule Helix.Process.Resources.Behaviour.KV do res_per_share = __MODULE__.div(resources, shares) - res_per_share >= 0 && res_per_share || 0.0 + # Ensure we do not return any negative or invalid number + map(res_per_share, fn v -> is_number(v) && v >= 0 && v || 0.0 end) end # At least currently, only local process may allocate static resources # This means that e.g. a FileDownload may consume RAM on the local server # but none on the remote one. + @spec allocate_static(Process.t) :: + t allocate_static(%{local?: false}) do initial() end @@ -179,6 +220,8 @@ defmodule Helix.Process.Resources.Behaviour.KV do end end + @spec allocate_dynamic(t, t, Process.t) :: + t allocate_dynamic(shares, res_per_share, process) do dynamic = Process.get_dynamic(process) @@ -191,10 +234,14 @@ defmodule Helix.Process.Resources.Behaviour.KV do end end + @spec allocate(t, t) :: + t allocate(dynamic_alloc, static_alloc) do sum(dynamic_alloc, static_alloc) end + @spec completed?(t, t) :: + t def completed?(processed, objective) do Enum.reduce(processed, %{}, fn {key, value}, acc -> # If the corresponding objective is `nil`, then by definition this @@ -212,6 +259,9 @@ defmodule Helix.Process.Resources.Behaviour.KV do end) end + @spec overflow?(t, [TOP.Allocator.allocated_process]) :: + {true, heaviest :: Process.t} + | false def overflow?(res, allocated_processes) do overflowed? = reduce(res, false, fn acc, val -> @@ -230,6 +280,8 @@ defmodule Helix.Process.Resources.Behaviour.KV do end end + @spec find_heaviest([TOP.Allocator.allocated_process]) :: + Process.t defp find_heaviest(allocated_processes) do allocated_processes |> Enum.sort_by(fn {process, resources} -> @@ -241,9 +293,13 @@ defmodule Helix.Process.Resources.Behaviour.KV do |> elem(0) end + @spec get_key(Process.t) :: + key defp get_key(process), do: Map.fetch!(process, @key) + @spec get_keys(t, t) :: + [key] defp get_keys(a, b) do Enum.uniq(Map.keys(a) ++ Map.keys(b)) end diff --git a/lib/process/resources/utils.ex b/lib/process/resources/utils.ex index 45eeba86..1bafdd67 100644 --- a/lib/process/resources/utils.ex +++ b/lib/process/resources/utils.ex @@ -1,10 +1,15 @@ defmodule Helix.Process.Resources.Utils do + @spec ensure_float(term) :: + float def ensure_float(i) when is_number(i), do: i / 1 |> Float.round(3) def ensure_float(map) when map_size(map) == 0, do: 0.0 + @spec safe_div(number, number, initial :: term) :: + number + | initial :: term def safe_div(dividend, divisor, _initial) when divisor > 0, do: dividend / divisor def safe_div(_, 0.0, initial), diff --git a/lib/server/model/server.ex b/lib/server/model/server.ex index 74b3175a..13548d4d 100644 --- a/lib/server/model/server.ex +++ b/lib/server/model/server.ex @@ -9,6 +9,7 @@ defmodule Helix.Server.Model.Server do alias HELL.Constant alias HELL.Password alias Helix.Hardware.Model.Component + alias Helix.Hardware.Model.Motherboard alias Helix.Server.Model.ServerType @type password :: String.t @@ -22,6 +23,8 @@ defmodule Helix.Server.Model.Server do updated_at: NaiveDateTime.t } + @type resources :: Motherboard.resources + @type creation_params :: %{ :server_type => Constant.t, optional(:motherboard_id) => Component.idtb | nil diff --git a/lib/software/model/software_type/firewall/process_type.ex b/lib/software/model/software_type/firewall/process_type.ex index 6ce035c1..bc64b061 100644 --- a/lib/software/model/software_type/firewall/process_type.ex +++ b/lib/software/model/software_type/firewall/process_type.ex @@ -10,35 +10,12 @@ defmodule Helix.Software.Model.SoftwareType.Firewall.Passive do defimpl Helix.Process.Model.Processable do - alias Helix.Software.Event.Firewall.Started, as: FirewallStartedEvent alias Helix.Software.Event.Firewall.Stopped, as: FirewallStoppedEvent - @ram_base_factor 5 - @cpu_base_factor 2 - - def dynamic_resources(_), - do: [] - - def minimum(%{version: v}), - do: %{ - paused: %{ - ram: v * @ram_base_factor - }, - running: %{ - ram: v * @ram_base_factor, - cpu: v * @cpu_base_factor - } - } - def kill(data, process, _) do - process = - process - |> Ecto.Changeset.change() - |> Map.put(:action, :delete) - event = %FirewallStoppedEvent{ version: data.version, - gateway_id: Ecto.Changeset.get_field(process, :gateway_id) + gateway_id: process.gateway_id } {:delete, [event]} @@ -47,7 +24,7 @@ defmodule Helix.Software.Model.SoftwareType.Firewall.Passive do def complete(data, process) do event = %FirewallStoppedEvent{ version: data.version, - gateway_id: Ecto.Changeset.get_field(process, :gateway_id) + gateway_id: process.gateway_id } {:delete, [event]} @@ -57,21 +34,6 @@ defmodule Helix.Software.Model.SoftwareType.Firewall.Passive do {:delete, []} end - def state_change(data, process, :paused, :running) do - event = %FirewallStartedEvent{ - version: data.version, - gateway_id: Ecto.Changeset.get_field(process, :gateway_id) - } - - {process, [event]} - end - - def state_change(_, process, _, _), - do: {process, []} - - def conclusion(_, _), - do: raise "firewall(passive) process should not be 'completed'" - def after_read_hook(data), do: data end diff --git a/lib/software/process/cracker/bruteforce.ex b/lib/software/process/cracker/bruteforce.ex index 41d97119..cf923539 100644 --- a/lib/software/process/cracker/bruteforce.ex +++ b/lib/software/process/cracker/bruteforce.ex @@ -19,7 +19,7 @@ process Helix.Software.Process.Cracker.Bruteforce do target_server_ip: Network.ip } - @typep creation_params :: + @type creation_params :: %{ target_server_ip: Network.ip } @@ -30,10 +30,11 @@ process Helix.Software.Process.Cracker.Bruteforce do %{ objective: objective, static: map, - dynamic: [:cpu] + l_dynamic: [:cpu], + r_dynamic: [] } - @typep resources_params :: + @type resources_params :: %{ cracker: File.t_of_type(:cracker), hasher: File.t_of_type(:hasher) | nil @@ -84,7 +85,7 @@ process Helix.Software.Process.Cracker.Bruteforce do alias Helix.Software.Model.File alias Helix.Software.Process.Cracker.Bruteforce, as: BruteforceProcess - @type params :: BruteforceProcess.objective_params + @type params :: BruteforceProcess.resources_params @type factors :: %{ diff --git a/lib/software/process/cracker/overflow.ex b/lib/software/process/cracker/overflow.ex index 6f1abe60..83e1ecd4 100644 --- a/lib/software/process/cracker/overflow.ex +++ b/lib/software/process/cracker/overflow.ex @@ -15,7 +15,7 @@ process Helix.Software.Process.Cracker.Overflow do target_connection_id: Connection.id | nil } - @typep creation_params :: + @type creation_params :: %{ target_process_id: Process.id | nil, target_connection_id: Connection.id | nil @@ -27,10 +27,11 @@ process Helix.Software.Process.Cracker.Overflow do %{ objective: objective, static: map, - dynamic: [:cpu] + l_dynamic: [:cpu], + r_dynamic: [] } - @typep resources_params :: + @type resources_params :: %{ cracker: File.t } @@ -90,7 +91,7 @@ process Helix.Software.Process.Cracker.Overflow do alias Helix.Software.Model.File alias Helix.Software.Process.Cracker.Overflow, as: OverflowProcess - @type params :: OverflowProcess.objective_params + @type params :: OverflowProcess.resources_params @type factors :: %{ cracker: %{version: FileFactor.fact_version} diff --git a/lib/software/process/file/transfer.ex b/lib/software/process/file/transfer.ex index 60a5a2d9..d9fda757 100644 --- a/lib/software/process/file/transfer.ex +++ b/lib/software/process/file/transfer.ex @@ -26,7 +26,8 @@ process Helix.Software.Process.File.Transfer do @type resources :: %{ objective: objective, - dynamic: [:dlk] | [:ulk], + l_dynamic: [:dlk] | [:ulk], + r_dynamic: [:ulk] | [:dlk], static: map } @@ -37,13 +38,13 @@ process Helix.Software.Process.File.Transfer do @type transfer_type :: :download | :upload @type connection_type :: :ftp | :public_ftp - @typep creation_params :: %{ + @type creation_params :: %{ type: transfer_type, connection_type: connection_type, destination_storage_id: Storage.id } - @typep resources_params :: %{ + @type resources_params :: %{ type: transfer_type, file: File.t, network_id: Network.id @@ -129,9 +130,10 @@ process Helix.Software.Process.File.Transfer do Sets the objectives to FileTransferProcess """ + alias Helix.Software.Process.File.Transfer, as: FileTransferProcess alias Helix.Software.Factor.File, as: FileFactor - @type params :: FileFactor.resources_params + @type params :: FileTransferProcess.resources_params @type factors :: %{ diff --git a/lib/universe/bank/process/bank/account/password_reveal.ex b/lib/universe/bank/process/bank/account/password_reveal.ex index cadda169..dbb9a2f7 100644 --- a/lib/universe/bank/process/bank/account/password_reveal.ex +++ b/lib/universe/bank/process/bank/account/password_reveal.ex @@ -28,10 +28,11 @@ process Helix.Universe.Bank.Process.Bank.Account.RevealPassword do @type resources :: %{ objective: objective, static: map, - dynamic: [:cpu] + l_dynamic: [:cpu], + r_dynamic: [] } - @typep resources_params :: + @type resources_params :: %{ account: BankAccount.t } @@ -68,7 +69,7 @@ process Helix.Universe.Bank.Process.Bank.Account.RevealPassword do alias Helix.Universe.Bank.Process.Bank.Account.RevealPassword, as: RevealPasswordProcess - @type params :: RevealPasswordProcess.objective_params + @type params :: RevealPasswordProcess.resources_params @type factors :: term # TODO proper balance diff --git a/lib/universe/bank/process/bank/transfer.ex b/lib/universe/bank/process/bank/transfer.ex index e7c4bfe7..ed3f6f53 100644 --- a/lib/universe/bank/process/bank/transfer.ex +++ b/lib/universe/bank/process/bank/transfer.ex @@ -14,7 +14,7 @@ process Helix.Universe.Bank.Process.Bank.Transfer do amount: BankTransfer.amount } - @typep creation_params :: + @type creation_params :: %{ transfer: BankTransfer.t } @@ -24,10 +24,11 @@ process Helix.Universe.Bank.Process.Bank.Transfer do @type resources :: %{ objective: objective, static: map, - dynamic: [] + l_dynamic: [], + r_dynamic: [] } - @typep resources_params :: + @type resources_params :: %{ transfer: BankTransfer.t } @@ -73,7 +74,7 @@ process Helix.Universe.Bank.Process.Bank.Transfer do alias Helix.Universe.Bank.Process.Bank.Transfer, as: BankTransferProcess - @type params :: BankTransferProcess.objective_params + @type params :: BankTransferProcess.resources_params @type factors :: term get_factors(%{transfer: _}) do end diff --git a/test/entity/event/handler/database_test.exs b/test/entity/event/handler/database_test.exs index e46c7df0..61029931 100644 --- a/test/entity/event/handler/database_test.exs +++ b/test/entity/event/handler/database_test.exs @@ -32,9 +32,11 @@ defmodule Helix.Entity.Event.Handler.DatabaseTest do on_db = DatabaseQuery.fetch_bank_account(entry.entity_id, acc) assert on_db.password == password - assert on_db.last_update > entry.last_update refute on_db.token refute on_db.last_login_date + + diff = DateTime.diff(on_db.last_update, entry.last_update, :millisecond) + assert diff > 0 end test "a new entry is created in case it did not exist before" do @@ -54,8 +56,11 @@ defmodule Helix.Entity.Event.Handler.DatabaseTest do on_db = DatabaseQuery.fetch_bank_account(fake_entry.entity_id, acc) assert on_db.password == password - assert on_db.last_update > fake_entry.last_update refute on_db.last_login_date + + diff = + DateTime.diff(on_db.last_update, fake_entry.last_update, :millisecond) + assert diff > 0 end end @@ -71,9 +76,11 @@ defmodule Helix.Entity.Event.Handler.DatabaseTest do on_db = DatabaseQuery.fetch_bank_account(entry.entity_id, acc) assert on_db.token == token - assert on_db.last_update > entry.last_update refute on_db.password refute on_db.last_login_date + + diff = DateTime.diff(on_db.last_update, entry.last_update, :millisecond) + assert diff > 0 end test "a new entry is created in case it did not exist before" do @@ -89,9 +96,12 @@ defmodule Helix.Entity.Event.Handler.DatabaseTest do on_db = DatabaseQuery.fetch_bank_account(fake_entry.entity_id, acc) assert on_db.token == token - assert on_db.last_update > fake_entry.last_update refute on_db.password refute on_db.last_login_date + + diff = + DateTime.diff(on_db.last_update, fake_entry.last_update, :millisecond) + assert diff > 0 end end @@ -106,10 +116,12 @@ defmodule Helix.Entity.Event.Handler.DatabaseTest do on_db = DatabaseQuery.fetch_bank_account(entry.entity_id, acc) refute on_db.token - assert on_db.last_update > entry.last_update assert on_db.password == acc.password assert on_db.last_login_date assert on_db.known_balance == acc.balance + + diff = DateTime.diff(on_db.last_update, entry.last_update, :millisecond) + assert diff > 0 end test "a new entry is created in case it did not exist before" do @@ -124,10 +136,13 @@ defmodule Helix.Entity.Event.Handler.DatabaseTest do on_db = DatabaseQuery.fetch_bank_account(fake_entry.entity_id, acc) refute on_db.token - assert on_db.last_update > fake_entry.last_update assert on_db.password == acc.password assert on_db.last_login_date assert on_db.known_balance == acc.balance + + diff = + DateTime.diff(on_db.last_update, fake_entry.last_update, :millisecond) + assert diff > 0 end end end diff --git a/test/process/query/process_test.exs b/test/process/query/process_test.exs index fb281469..b653dc1c 100644 --- a/test/process/query/process_test.exs +++ b/test/process/query/process_test.exs @@ -88,7 +88,7 @@ defmodule Helix.Process.Query.ProcessTest do end test "returns empty list if no process is found" do - refute ProcessQuery.get_custom("file_download", Server.ID.generate(), %{}) + refute ProcessQuery.get_custom(:file_download, Server.ID.generate(), %{}) end end end