From ace16686059b6837f4ae11170704994f8db404ce Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sun, 29 Sep 2024 17:43:56 -0400 Subject: [PATCH 01/18] spike: incorporate namespaced injectable application --- .tool-versions | 4 +- bin/start | 5 - .formatter.exs => engine/.formatter.exs | 0 engine/.gitignore | 26 ++ engine/README.md | 21 ++ engine/after.ex | 275 +++++++++++++++ engine/before.ex | 272 +++++++++++++++ engine/lib/engine.ex | 2 + engine/lib/engine/application.ex | 19 + engine/lib/engine/document_symbols.ex | 241 +++++++++++++ engine/lib/engine/worker.ex | 64 ++++ engine/mix.exs | 35 ++ engine/mix.lock | 10 + engine/test/engine_test.exs | 8 + {test => engine/test}/test_helper.exs | 0 expert/.formatter.exs | 4 + .gitignore => expert/.gitignore | 0 README.md => expert/README.md | 0 expert/bin/start | 11 + {config => expert/config}/config.exs | 0 {config => expert/config}/dev.exs | 0 {config => expert/config}/prod.exs | 0 {config => expert/config}/test.exs | 0 expert/lib/expert.ex | 251 ++++++++++++++ {lib => expert/lib}/expert/application.ex | 0 {lib => expert/lib}/expert/lsp_supervisor.ex | 30 +- expert/lib/expert/release.ex | 27 ++ expert/lib/expert/runtime.ex | 328 ++++++++++++++++++ expert/lib/expert/runtime/supervisor.ex | 26 ++ mix.exs => expert/mix.exs | 9 +- mix.lock => expert/mix.lock | 2 +- expert/priv/cmd | 22 ++ {test => expert/test}/expert_test.exs | 0 expert/test/test_helper.exs | 1 + justfile | 40 ++- lib/expert.ex | 83 ----- namespace/.formatter.exs | 4 + namespace/.gitignore | 26 ++ namespace/README.md | 21 ++ namespace/lib/mix/tasks/namespace.ex | 149 ++++++++ namespace/lib/mix/tasks/namespace/abstract.ex | 297 ++++++++++++++++ namespace/lib/mix/tasks/namespace/code.ex | 5 + namespace/lib/mix/tasks/namespace/module.ex | 83 +++++ namespace/lib/mix/tasks/namespace/path.ex | 29 ++ .../namespace/transform/app_directories.ex | 25 ++ .../lib/mix/tasks/namespace/transform/apps.ex | 79 +++++ .../mix/tasks/namespace/transform/beams.ex | 117 +++++++ .../mix/tasks/namespace/transform/boots.ex | 26 ++ .../mix/tasks/namespace/transform/configs.ex | 41 +++ .../mix/tasks/namespace/transform/erlang.ex | 33 ++ .../mix/tasks/namespace/transform/scripts.ex | 99 ++++++ namespace/mix.exs | 28 ++ namespace/test/namespace_test.exs | 8 + namespace/test/test_helper.exs | 1 + 54 files changed, 2748 insertions(+), 139 deletions(-) delete mode 100755 bin/start rename .formatter.exs => engine/.formatter.exs (100%) create mode 100644 engine/.gitignore create mode 100644 engine/README.md create mode 100644 engine/after.ex create mode 100644 engine/before.ex create mode 100644 engine/lib/engine.ex create mode 100644 engine/lib/engine/application.ex create mode 100644 engine/lib/engine/document_symbols.ex create mode 100644 engine/lib/engine/worker.ex create mode 100644 engine/mix.exs create mode 100644 engine/mix.lock create mode 100644 engine/test/engine_test.exs rename {test => engine/test}/test_helper.exs (100%) create mode 100644 expert/.formatter.exs rename .gitignore => expert/.gitignore (100%) rename README.md => expert/README.md (100%) create mode 100755 expert/bin/start rename {config => expert/config}/config.exs (100%) rename {config => expert/config}/dev.exs (100%) rename {config => expert/config}/prod.exs (100%) rename {config => expert/config}/test.exs (100%) create mode 100644 expert/lib/expert.ex rename {lib => expert/lib}/expert/application.ex (100%) rename {lib => expert/lib}/expert/lsp_supervisor.ex (64%) create mode 100644 expert/lib/expert/release.ex create mode 100644 expert/lib/expert/runtime.ex create mode 100644 expert/lib/expert/runtime/supervisor.ex rename mix.exs => expert/mix.exs (77%) rename mix.lock => expert/mix.lock (88%) create mode 100755 expert/priv/cmd rename {test => expert/test}/expert_test.exs (100%) create mode 100644 expert/test/test_helper.exs delete mode 100644 lib/expert.ex create mode 100644 namespace/.formatter.exs create mode 100644 namespace/.gitignore create mode 100644 namespace/README.md create mode 100644 namespace/lib/mix/tasks/namespace.ex create mode 100644 namespace/lib/mix/tasks/namespace/abstract.ex create mode 100644 namespace/lib/mix/tasks/namespace/code.ex create mode 100644 namespace/lib/mix/tasks/namespace/module.ex create mode 100644 namespace/lib/mix/tasks/namespace/path.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/app_directories.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/apps.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/beams.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/boots.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/configs.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/erlang.ex create mode 100644 namespace/lib/mix/tasks/namespace/transform/scripts.ex create mode 100644 namespace/mix.exs create mode 100644 namespace/test/namespace_test.exs create mode 100644 namespace/test/test_helper.exs diff --git a/.tool-versions b/.tool-versions index b7c4c5bd..e8a5e4e7 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -elixir 1.17.2-otp-27 -erlang 27.0.1 +elixir 1.17.2-otp-26 +erlang 26.2.5 diff --git a/bin/start b/bin/start deleted file mode 100755 index ced599b7..00000000 --- a/bin/start +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -cd "$(dirname "$0")"/.. || exit 1 - -mix run --no-halt -e "Application.ensure_all_started(:expert)" -- "$@" diff --git a/.formatter.exs b/engine/.formatter.exs similarity index 100% rename from .formatter.exs rename to engine/.formatter.exs diff --git a/engine/.gitignore b/engine/.gitignore new file mode 100644 index 00000000..487459f8 --- /dev/null +++ b/engine/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +engine-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/engine/README.md b/engine/README.md new file mode 100644 index 00000000..1adedf77 --- /dev/null +++ b/engine/README.md @@ -0,0 +1,21 @@ +# Engine + +**TODO: Add description** + +## Installation + +If [available in Hex](https://hex.pm/docs/publish), the package can be installed +by adding `engine` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [ + {:engine, "~> 0.1.0"} + ] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at . + diff --git a/engine/after.ex b/engine/after.ex new file mode 100644 index 00000000..28738fa8 --- /dev/null +++ b/engine/after.ex @@ -0,0 +1,275 @@ +[ + {:attribute, 1, :file, {~c"lib/schematic/unification.ex", 1}}, + {:attribute, 1, :module, XPSchematic.Unification}, + {:attribute, 1, :compile, + [:no_auto_import, :debug_info, {:inline, [struct_impl_for: 1]}]}, + {:attribute, 4, :callback, + {{:unify, 3}, + [ + {:type, 4, :fun, + [ + {:type, 4, :product, + [ + {:user_type, 4, :t, []}, + {:type, 4, :term, []}, + {:type, 4, :term, []} + ]}, + {:type, 4, :term, []} + ]} + ]}}, + {:attribute, 5, :callback, + {{:message, 1}, + [ + {:type, 5, :fun, + [{:type, 5, :product, [{:user_type, 5, :t, []}]}, {:type, 5, :term, []}]} + ]}}, + {:attribute, 6, :callback, + {{:kind, 1}, + [ + {:type, 6, :fun, + [{:type, 6, :product, [{:user_type, 6, :t, []}]}, {:type, 6, :term, []}]} + ]}}, + {:attribute, 1, :spec, + {{:impl_for!, 1}, + [ + {:type, 1, :fun, + [{:type, 1, :product, [{:type, 1, :term, []}]}, {:type, 1, :atom, []}]} + ]}}, + {:attribute, 1, :spec, + {{:impl_for, 1}, + [ + {:type, 1, :fun, + [ + {:type, 1, :product, [{:type, 1, :term, []}]}, + {:type, 1, :union, [{:type, 1, :atom, []}, {:atom, 0, nil}]} + ]} + ]}}, + {:attribute, 1, :spec, + {{:__protocol__, 1}, + [ + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :module}]}, + {:atom, 0, XPSchematic.Unification} + ]}, + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :functions}]}, + {:type, 0, :nonempty_list, + [ + {:type, 0, :union, + [ + {:type, 0, :tuple, [{:atom, 0, :unify}, {:integer, 0, 3}]}, + {:type, 0, :tuple, [{:atom, 0, :message}, {:integer, 0, 1}]}, + {:type, 0, :tuple, [{:atom, 0, :kind}, {:integer, 0, 1}]} + ]} + ]} + ]}, + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :consolidated?}]}, + {:type, 1, :boolean, []} + ]}, + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :impls}]}, + {:type, 1, :union, + [ + {:atom, 0, :not_consolidated}, + {:type, 0, :tuple, + [ + {:atom, 0, :consolidated}, + {:type, 0, :list, [{:type, 1, :module, []}]} + ]} + ]} + ]} + ]}}, + {:attribute, 1, :export_type, [t: 0]}, + {:attribute, 1, :type, {:t, {:type, 1, :term, []}, []}}, + {:attribute, 1, :dialyzer, + {:nowarn_function, [__protocol__: 1, impl_for: 1, impl_for!: 1]}}, + {:attribute, 1, :__protocol__, [fallback_to_any: true]}, + {:attribute, 1, :export, + [ + __info__: 1, + __protocol__: 1, + impl_for: 1, + impl_for!: 1, + kind: 1, + message: 1, + unify: 3 + ]}, + {:attribute, 1, :spec, + {{:__info__, 1}, + [ + {:type, 1, :fun, + [ + {:type, 1, :product, + [ + {:type, 1, :union, + [ + {:atom, 1, :attributes}, + {:atom, 1, :compile}, + {:atom, 1, :functions}, + {:atom, 1, :macros}, + {:atom, 1, :md5}, + {:atom, 1, :exports_md5}, + {:atom, 1, :module}, + {:atom, 1, :deprecated}, + {:atom, 1, :struct} + ]} + ]}, + {:type, 1, :any, []} + ]} + ]}}, + {:function, 0, :__info__, 1, + [ + {:clause, 0, [{:atom, 0, :module}], [], + [{:atom, 0, XPSchematic.Unification}]}, + {:clause, 0, [{:atom, 0, :functions}], [], + [ + {:cons, 0, {:tuple, 0, [{:atom, 0, :__protocol__}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for!}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :kind}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :message}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :unify}, {:integer, 0, 3}]}, + {nil, 0}}}}}}} + ]}, + {:clause, 0, [{:atom, 0, :macros}], [], [nil: 0]}, + {:clause, 0, [{:atom, 0, :struct}], [], [{:atom, 0, nil}]}, + {:clause, 0, [{:atom, 0, :exports_md5}], [], + [ + {:bin, 0, + [ + {:bin_element, 0, + {:string, 0, + [89, 58, 13, 228, 237, 62, 86, 234, 224, 87, 49, 205, 30, 99, 0, + 126]}, :default, :default} + ]} + ]}, + {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :attributes}}], [], + [ + {:call, 0, + {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, + [{:atom, 0, XPSchematic.Unification}, {:var, 0, :Key}]} + ]}, + {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :compile}}], [], + [ + {:call, 0, + {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, + [{:atom, 0, XPSchematic.Unification}, {:var, 0, :Key}]} + ]}, + {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :md5}}], [], + [ + {:call, 0, + {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, + [{:atom, 0, XPSchematic.Unification}, {:var, 0, :Key}]} + ]}, + {:clause, 0, [{:atom, 0, :deprecated}], [], [nil: 0]} + ]}, + {:function, 1, :impl_for!, 1, + [ + {:clause, [generated: true, location: 0], [{:var, 1, :_@1}], [], + [{:call, 1, {:atom, 1, :impl_for}, [{:var, 1, :_@1}]}]} + ]}, + {:function, 6, :kind, 1, + [ + {:clause, 6, [{:var, 6, :_@1}], [], + [ + {:call, 6, + {:remote, 6, {:call, 6, {:atom, 6, :impl_for!}, [{:var, 6, :_@1}]}, + {:atom, 6, :kind}}, [{:var, 6, :_@1}]} + ]} + ]}, + {:function, 5, :message, 1, + [ + {:clause, 5, [{:var, 5, :_@1}], [], + [ + {:call, 5, + {:remote, 5, {:call, 5, {:atom, 5, :impl_for!}, [{:var, 5, :_@1}]}, + {:atom, 5, :message}}, [{:var, 5, :_@1}]} + ]} + ]}, + {:function, 4, :unify, 3, + [ + {:clause, 4, [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}], [], + [ + {:call, 4, + {:remote, 4, {:call, 4, {:atom, 4, :impl_for!}, [{:var, 4, :_@1}]}, + {:atom, 4, :unify}}, + [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}]} + ]} + ]}, + {:function, 1, :struct_impl_for, 1, + [ + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], XPSchematic}], [], + [ + {:atom, [generated: true, location: 0], + XPSchematic.Unification.Schematic} + ]}, + {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], + [{:atom, [generated: true, location: 0], XPSchematic.Unification.Any}]} + ]}, + {:function, 1, :impl_for, 1, + [ + {:clause, [generated: true, location: 0], + [ + {:map, 1, + [{:map_field_exact, 1, {:atom, 1, :__struct__}, {:var, 1, :_@1}}]} + ], + [ + [ + {:call, 1, {:remote, 1, {:atom, 1, :erlang}, {:atom, 1, :is_atom}}, + [{:var, 1, :_@1}]} + ] + ], [{:call, 1, {:atom, 1, :struct_impl_for}, [{:var, 1, :_@1}]}]}, + {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], + [{:atom, [generated: true, location: 0], XPSchematic.Unification.Any}]} + ]}, + {:function, 1, :__protocol__, 1, + [ + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :module}], [], + [{:atom, [generated: true, location: 0], XPSchematic.Unification}]}, + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :functions}], [], + [ + {:cons, [generated: true, location: 0], + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :kind}, + {:integer, [generated: true, location: 0], 1} + ]}, + {:cons, [generated: true, location: 0], + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :message}, + {:integer, [generated: true, location: 0], 1} + ]}, + {:cons, [generated: true, location: 0], + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :unify}, + {:integer, [generated: true, location: 0], 3} + ]}, {nil, [generated: true, location: 0]}}}} + ]}, + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :consolidated?}], [], + [{:atom, [generated: true, location: 0], true}]}, + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :impls}], [], + [ + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :consolidated}, + {:cons, [generated: true, location: 0], + {:atom, [generated: true, location: 0], Any}, + {:cons, [generated: true, location: 0], + {:atom, [generated: true, location: 0], XPSchematic}, + {nil, [generated: true, location: 0]}}} + ]} + ]} + ]} +] \ No newline at end of file diff --git a/engine/before.ex b/engine/before.ex new file mode 100644 index 00000000..5d2122b1 --- /dev/null +++ b/engine/before.ex @@ -0,0 +1,272 @@ +[ + {:attribute, 1, :file, {~c"lib/schematic/unification.ex", 1}}, + {:attribute, 1, :module, Schematic.Unification}, + {:attribute, 1, :compile, + [:no_auto_import, :debug_info, {:inline, [struct_impl_for: 1]}]}, + {:attribute, 4, :callback, + {{:unify, 3}, + [ + {:type, 4, :fun, + [ + {:type, 4, :product, + [ + {:user_type, 4, :t, []}, + {:type, 4, :term, []}, + {:type, 4, :term, []} + ]}, + {:type, 4, :term, []} + ]} + ]}}, + {:attribute, 5, :callback, + {{:message, 1}, + [ + {:type, 5, :fun, + [{:type, 5, :product, [{:user_type, 5, :t, []}]}, {:type, 5, :term, []}]} + ]}}, + {:attribute, 6, :callback, + {{:kind, 1}, + [ + {:type, 6, :fun, + [{:type, 6, :product, [{:user_type, 6, :t, []}]}, {:type, 6, :term, []}]} + ]}}, + {:attribute, 1, :spec, + {{:impl_for!, 1}, + [ + {:type, 1, :fun, + [{:type, 1, :product, [{:type, 1, :term, []}]}, {:type, 1, :atom, []}]} + ]}}, + {:attribute, 1, :spec, + {{:impl_for, 1}, + [ + {:type, 1, :fun, + [ + {:type, 1, :product, [{:type, 1, :term, []}]}, + {:type, 1, :union, [{:type, 1, :atom, []}, {:atom, 0, nil}]} + ]} + ]}}, + {:attribute, 1, :spec, + {{:__protocol__, 1}, + [ + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :module}]}, + {:atom, 0, Schematic.Unification} + ]}, + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :functions}]}, + {:type, 0, :nonempty_list, + [ + {:type, 0, :union, + [ + {:type, 0, :tuple, [{:atom, 0, :unify}, {:integer, 0, 3}]}, + {:type, 0, :tuple, [{:atom, 0, :message}, {:integer, 0, 1}]}, + {:type, 0, :tuple, [{:atom, 0, :kind}, {:integer, 0, 1}]} + ]} + ]} + ]}, + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :consolidated?}]}, + {:type, 1, :boolean, []} + ]}, + {:type, 1, :fun, + [ + {:type, 1, :product, [{:atom, 0, :impls}]}, + {:type, 1, :union, + [ + {:atom, 0, :not_consolidated}, + {:type, 0, :tuple, + [ + {:atom, 0, :consolidated}, + {:type, 0, :list, [{:type, 1, :module, []}]} + ]} + ]} + ]} + ]}}, + {:attribute, 1, :export_type, [t: 0]}, + {:attribute, 1, :type, {:t, {:type, 1, :term, []}, []}}, + {:attribute, 1, :dialyzer, + {:nowarn_function, [__protocol__: 1, impl_for: 1, impl_for!: 1]}}, + {:attribute, 1, :__protocol__, [fallback_to_any: true]}, + {:attribute, 1, :export, + [ + __info__: 1, + __protocol__: 1, + impl_for: 1, + impl_for!: 1, + kind: 1, + message: 1, + unify: 3 + ]}, + {:attribute, 1, :spec, + {{:__info__, 1}, + [ + {:type, 1, :fun, + [ + {:type, 1, :product, + [ + {:type, 1, :union, + [ + {:atom, 1, :attributes}, + {:atom, 1, :compile}, + {:atom, 1, :functions}, + {:atom, 1, :macros}, + {:atom, 1, :md5}, + {:atom, 1, :exports_md5}, + {:atom, 1, :module}, + {:atom, 1, :deprecated}, + {:atom, 1, :struct} + ]} + ]}, + {:type, 1, :any, []} + ]} + ]}}, + {:function, 0, :__info__, 1, + [ + {:clause, 0, [{:atom, 0, :module}], [], + [{:atom, 0, Schematic.Unification}]}, + {:clause, 0, [{:atom, 0, :functions}], [], + [ + {:cons, 0, {:tuple, 0, [{:atom, 0, :__protocol__}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for!}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :kind}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :message}, {:integer, 0, 1}]}, + {:cons, 0, {:tuple, 0, [{:atom, 0, :unify}, {:integer, 0, 3}]}, + {nil, 0}}}}}}} + ]}, + {:clause, 0, [{:atom, 0, :macros}], [], [nil: 0]}, + {:clause, 0, [{:atom, 0, :struct}], [], [{:atom, 0, nil}]}, + {:clause, 0, [{:atom, 0, :exports_md5}], [], + [ + {:bin, 0, + [ + {:bin_element, 0, + {:string, 0, + [89, 58, 13, 228, 237, 62, 86, 234, 224, 87, 49, 205, 30, 99, 0, + 126]}, :default, :default} + ]} + ]}, + {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :attributes}}], [], + [ + {:call, 0, + {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, + [{:atom, 0, Schematic.Unification}, {:var, 0, :Key}]} + ]}, + {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :compile}}], [], + [ + {:call, 0, + {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, + [{:atom, 0, Schematic.Unification}, {:var, 0, :Key}]} + ]}, + {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :md5}}], [], + [ + {:call, 0, + {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, + [{:atom, 0, Schematic.Unification}, {:var, 0, :Key}]} + ]}, + {:clause, 0, [{:atom, 0, :deprecated}], [], [nil: 0]} + ]}, + {:function, 1, :impl_for!, 1, + [ + {:clause, [generated: true, location: 0], [{:var, 1, :_@1}], [], + [{:call, 1, {:atom, 1, :impl_for}, [{:var, 1, :_@1}]}]} + ]}, + {:function, 6, :kind, 1, + [ + {:clause, 6, [{:var, 6, :_@1}], [], + [ + {:call, 6, + {:remote, 6, {:call, 6, {:atom, 6, :impl_for!}, [{:var, 6, :_@1}]}, + {:atom, 6, :kind}}, [{:var, 6, :_@1}]} + ]} + ]}, + {:function, 5, :message, 1, + [ + {:clause, 5, [{:var, 5, :_@1}], [], + [ + {:call, 5, + {:remote, 5, {:call, 5, {:atom, 5, :impl_for!}, [{:var, 5, :_@1}]}, + {:atom, 5, :message}}, [{:var, 5, :_@1}]} + ]} + ]}, + {:function, 4, :unify, 3, + [ + {:clause, 4, [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}], [], + [ + {:call, 4, + {:remote, 4, {:call, 4, {:atom, 4, :impl_for!}, [{:var, 4, :_@1}]}, + {:atom, 4, :unify}}, + [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}]} + ]} + ]}, + {:function, 1, :struct_impl_for, 1, + [ + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], Schematic}], [], + [{:atom, [generated: true, location: 0], Schematic.Unification.Schematic}]}, + {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], + [{:atom, [generated: true, location: 0], Schematic.Unification.Any}]} + ]}, + {:function, 1, :impl_for, 1, + [ + {:clause, [generated: true, location: 0], + [ + {:map, 1, + [{:map_field_exact, 1, {:atom, 1, :__struct__}, {:var, 1, :_@1}}]} + ], + [ + [ + {:call, 1, {:remote, 1, {:atom, 1, :erlang}, {:atom, 1, :is_atom}}, + [{:var, 1, :_@1}]} + ] + ], [{:call, 1, {:atom, 1, :struct_impl_for}, [{:var, 1, :_@1}]}]}, + {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], + [{:atom, [generated: true, location: 0], Schematic.Unification.Any}]} + ]}, + {:function, 1, :__protocol__, 1, + [ + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :module}], [], + [{:atom, [generated: true, location: 0], Schematic.Unification}]}, + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :functions}], [], + [ + {:cons, [generated: true, location: 0], + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :kind}, + {:integer, [generated: true, location: 0], 1} + ]}, + {:cons, [generated: true, location: 0], + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :message}, + {:integer, [generated: true, location: 0], 1} + ]}, + {:cons, [generated: true, location: 0], + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :unify}, + {:integer, [generated: true, location: 0], 3} + ]}, {nil, [generated: true, location: 0]}}}} + ]}, + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :consolidated?}], [], + [{:atom, [generated: true, location: 0], true}]}, + {:clause, [generated: true, location: 0], + [{:atom, [generated: true, location: 0], :impls}], [], + [ + {:tuple, [generated: true, location: 0], + [ + {:atom, [generated: true, location: 0], :consolidated}, + {:cons, [generated: true, location: 0], + {:atom, [generated: true, location: 0], Any}, + {:cons, [generated: true, location: 0], + {:atom, [generated: true, location: 0], Schematic}, + {nil, [generated: true, location: 0]}}} + ]} + ]} + ]} +] \ No newline at end of file diff --git a/engine/lib/engine.ex b/engine/lib/engine.ex new file mode 100644 index 00000000..115929d9 --- /dev/null +++ b/engine/lib/engine.ex @@ -0,0 +1,2 @@ +defmodule Engine do +end diff --git a/engine/lib/engine/application.ex b/engine/lib/engine/application.ex new file mode 100644 index 00000000..fd897802 --- /dev/null +++ b/engine/lib/engine/application.ex @@ -0,0 +1,19 @@ +defmodule Engine.Application do + # See https://hexdocs.pm/elixir/Application.html + # for more information on OTP Applications + @moduledoc false + + use Application + + @impl true + def start(_type, _args) do + children = [ + Engine.Worker + ] + + # See https://hexdocs.pm/elixir/Supervisor.html + # for other strategies and supported options + opts = [strategy: :one_for_one, name: Engine.Supervisor] + Supervisor.start_link(children, opts) + end +end diff --git a/engine/lib/engine/document_symbols.ex b/engine/lib/engine/document_symbols.ex new file mode 100644 index 00000000..1eacfcfa --- /dev/null +++ b/engine/lib/engine/document_symbols.ex @@ -0,0 +1,241 @@ +defmodule Engine.DocumentSymbol do + @moduledoc false + + alias GenLSP.Enumerations.SymbolKind + alias GenLSP.Structures.DocumentSymbol + alias GenLSP.Structures.Position + alias GenLSP.Structures.Range + + @spec fetch(text :: String.t()) :: list(DocumentSymbol.t()) + def fetch(text) do + ast = + case Spitfire.parse( + text, + # we set the literal encoder so that we can know when atoms and strings start and end + # this makes it useful for knowing the exact locations of struct field definitions + literal_encoder: fn literal, meta -> + if is_atom(literal) or is_binary(literal) do + {:ok, {:__literal__, meta, [literal]}} + else + {:ok, literal} + end + end, + unescape: false, + token_metadata: true, + columns: true + ) do + {:error, ast, _errors} -> + ast + + {:error, _} -> + raise "Failed to parse!" + + {:ok, ast} -> + ast + end + + for %DocumentSymbol{} = ds <- List.wrap(walker(ast, nil)) do + {:ok, dumped} = Schematic.dump(DocumentSymbol.schema(), ds) + + dumped + end + end + + defp walker([{{:__literal__, _, [:do]}, {_, _, _exprs} = ast}], mod) do + walker(ast, mod) + end + + defp walker({:__block__, _, exprs}, mod) do + for expr <- exprs, sym = walker(expr, mod), sym != nil do + sym + end + end + + defp walker({:defmodule, meta, [name | children]}, _mod) do + name = Macro.to_string(unliteral(name)) + + %DocumentSymbol{ + name: name, + kind: SymbolKind.module(), + children: + List.flatten(for(child <- children, sym = walker(child, name), sym != nil, do: sym)), + range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:end][:line] - 1, character: meta[:end][:column] - 1} + }, + selection_range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:line] - 1, character: meta[:column] - 1} + } + } + end + + defp walker({:describe, meta, [name | children]}, mod) do + name = String.replace("describe " <> Macro.to_string(unliteral(name)), "\n", "") + + %DocumentSymbol{ + name: name, + kind: SymbolKind.class(), + children: + List.flatten(for(child <- children, sym = walker(child, mod), sym != nil, do: sym)), + range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:end][:line] - 1, character: meta[:end][:column] - 1} + }, + selection_range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:line] - 1, character: meta[:column] - 1} + } + } + end + + defp walker({:defstruct, meta, [fields]}, mod) do + fields = + for field <- fields do + {name, start_line, start_column} = + case field do + {:__literal__, meta, [name]} -> + start_line = meta[:line] - 1 + start_column = meta[:column] - 1 + name = Macro.to_string(name) + + {name, start_line, start_column} + + {{:__literal__, meta, [name]}, default} -> + start_line = meta[:line] - 1 + start_column = meta[:column] - 1 + name = to_string(name) <> ": " <> Macro.to_string(unliteral(default)) + + {name, start_line, start_column} + end + + %DocumentSymbol{ + name: name, + children: [], + kind: SymbolKind.field(), + range: %Range{ + start: %Position{ + line: start_line, + character: start_column + }, + end: %Position{ + line: start_line, + character: start_column + String.length(name) + } + }, + selection_range: %Range{ + start: %Position{line: start_line, character: start_column}, + end: %Position{line: start_line, character: start_column} + } + } + end + + %DocumentSymbol{ + name: "%#{mod}{}", + children: fields, + kind: elixir_kind_to_lsp_kind(:defstruct), + range: %Range{ + start: %Position{ + line: meta[:line] - 1, + character: meta[:column] - 1 + }, + end: %Position{ + line: (meta[:end_of_expression][:line] || meta[:line]) - 1, + character: (meta[:end_of_expression][:column] || meta[:column]) - 1 + } + }, + selection_range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:line] - 1, character: meta[:column] - 1} + } + } + end + + defp walker({:@, meta, [{_name, _, value}]} = attribute, _) when length(value) > 0 do + %DocumentSymbol{ + name: attribute |> unliteral() |> Macro.to_string() |> String.replace("\n", ""), + children: [], + kind: elixir_kind_to_lsp_kind(:@), + range: %Range{ + start: %Position{ + line: meta[:line] - 1, + character: meta[:column] - 1 + }, + end: %Position{ + line: (meta[:end_of_expression] || meta)[:line] - 1, + character: (meta[:end_of_expression] || meta)[:column] - 1 + } + }, + selection_range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:line] - 1, character: meta[:column] - 1} + } + } + end + + defp walker({type, meta, [name | _children]}, _) when type in [:test, :feature, :property] do + %DocumentSymbol{ + name: String.replace("#{type} #{Macro.to_string(unliteral(name))}", "\n", ""), + children: [], + kind: SymbolKind.constructor(), + range: %Range{ + start: %Position{ + line: meta[:line] - 1, + character: meta[:column] - 1 + }, + end: %Position{ + line: (meta[:end] || meta[:end_of_expression] || meta)[:line] - 1, + character: (meta[:end] || meta[:end_of_expression] || meta)[:column] - 1 + } + }, + selection_range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:line] - 1, character: meta[:column] - 1} + } + } + end + + defp walker({type, meta, [name | _children]}, _) + when type in [:def, :defp, :defmacro, :defmacro] do + %DocumentSymbol{ + name: String.replace("#{type} #{name |> unliteral() |> Macro.to_string()}", "\n", ""), + children: [], + kind: elixir_kind_to_lsp_kind(type), + range: %Range{ + start: %Position{ + line: meta[:line] - 1, + character: meta[:column] - 1 + }, + end: %Position{ + line: (meta[:end] || meta[:end_of_expression] || meta)[:line] - 1, + character: (meta[:end] || meta[:end_of_expression] || meta)[:column] - 1 + } + }, + selection_range: %Range{ + start: %Position{line: meta[:line] - 1, character: meta[:column] - 1}, + end: %Position{line: meta[:line] - 1, character: meta[:column] - 1} + } + } + end + + defp walker(_ast, _) do + nil + end + + defp unliteral(ast) do + Macro.prewalk(ast, fn + {:__literal__, _, [literal]} -> + literal + + node -> + node + end) + end + + defp elixir_kind_to_lsp_kind(:defstruct), do: SymbolKind.struct() + defp elixir_kind_to_lsp_kind(:@), do: SymbolKind.property() + + defp elixir_kind_to_lsp_kind(kind) + when kind in [:def, :defp, :defmacro, :defmacrop, :test, :describe], + do: SymbolKind.function() +end diff --git a/engine/lib/engine/worker.ex b/engine/lib/engine/worker.ex new file mode 100644 index 00000000..92817add --- /dev/null +++ b/engine/lib/engine/worker.ex @@ -0,0 +1,64 @@ +defmodule Engine.Worker do + use GenServer + + def start_link(arg) do + GenServer.start_link(__MODULE__, arg, name: __MODULE__) + end + + @impl GenServer + def init(_arg) do + working_dir = File.cwd!() + {:ok, %{working_dir: working_dir}} + end + + def enqueue_compiler(opts) do + GenServer.cast(__MODULE__, {:compile, opts}) + end + + defp flush(acc) do + receive do + {:"$gen_cast", {:compile, opts}} -> flush([opts | acc]) + after + 0 -> acc + end + end + + @impl GenServer + def handle_cast({:compile, opts}, state) do + # we essentially compile now and rollup any newer requests to compile, so that we aren't doing 5 compiles + # if we the user saves 5 times after saving one time + flush([]) + from = Keyword.fetch!(opts, :from) + + File.cd!(state.working_dir) + + result = Engine.Worker.compile() + + Process.send(from, {:compiler_result, result}, []) + {:noreply, state} + end + + def compile do + # keep stdout on this node + Process.group_leader(self(), Process.whereis(:user)) + + Mix.Task.clear() + + # load the paths for deps and compile them + # will noop if they are already compiled + # The mix cli basically runs this before any mix task + # we have to rerun because we already ran a mix task + # (mix run), which called this, but we also passed + # --no-compile, so nothing was compiled, but the + # task was not re-enabled it seems + Mix.Task.rerun("deps.loadpaths") + + Mix.Task.rerun("compile", [ + "--ignore-module-conflict", + "--no-protocol-consolidation", + "--return-errors" + ]) + rescue + e -> {:error, e} + end +end diff --git a/engine/mix.exs b/engine/mix.exs new file mode 100644 index 00000000..256f5b4c --- /dev/null +++ b/engine/mix.exs @@ -0,0 +1,35 @@ +defmodule Engine.MixProject do + use Mix.Project + + def project do + [ + app: :engine, + version: "0.1.0", + elixir: "~> 1.17", + start_permanent: Mix.env() == :prod, + aliases: [ + build: ["cmd rm -rf _build/#{Mix.env()}", "compile", "namespace _build/#{Mix.env()}"] + ], + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger], + mod: {Engine.Application, []} + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:spitfire, "~> 0.1"}, + # {:gen_lsp, "~> 0.10"}, + {:gen_lsp, + github: "elixir-tools/gen_lsp", branch: "change-schematic-function", override: true}, + {:namespace, path: "../namespace", only: [:dev, :prod], runtime: false} + ] + end +end diff --git a/engine/mix.lock b/engine/mix.lock new file mode 100644 index 00000000..91ee44f8 --- /dev/null +++ b/engine/mix.lock @@ -0,0 +1,10 @@ +%{ + "beam_file": {:hex, :beam_file, "0.6.2", "efd54ec60be6a03f0a8f96f72b0353427196613289c46032d3500f0ab6c34d32", [:mix], [], "hexpm", "09a99e8e5aad674edcad7213b0d7602375dfd3c7d02f8e3136e3efae0bcc9c56"}, + "gen_lsp": {:git, "https://github.com/elixir-tools/gen_lsp.git", "f63f284289ef61b678ab6bd2cbcf3a6ba3d9fcdd", [branch: "change-schematic-function"]}, + "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "schematic": {:hex, :schematic, "0.2.1", "0b091df94146fd15a0a343d1bd179a6c5a58562527746dadd09477311698dbb1", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0b255d65921e38006138201cd4263fd8bb807d9dfc511074615cd264a571b3b1"}, + "spitfire": {:hex, :spitfire, "0.1.3", "7ea0f544005dfbe48e615ed90250c9a271bfe126914012023fd5e4b6b82b7ec7", [:mix], [], "hexpm", "d53b5107bcff526a05c5bb54c95e77b36834550affd5830c9f58760e8c543657"}, + "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, + "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, +} diff --git a/engine/test/engine_test.exs b/engine/test/engine_test.exs new file mode 100644 index 00000000..1cb08fe3 --- /dev/null +++ b/engine/test/engine_test.exs @@ -0,0 +1,8 @@ +defmodule EngineTest do + use ExUnit.Case + doctest Engine + + test "greets the world" do + assert Engine.hello() == :world + end +end diff --git a/test/test_helper.exs b/engine/test/test_helper.exs similarity index 100% rename from test/test_helper.exs rename to engine/test/test_helper.exs diff --git a/expert/.formatter.exs b/expert/.formatter.exs new file mode 100644 index 00000000..d2cda26e --- /dev/null +++ b/expert/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/.gitignore b/expert/.gitignore similarity index 100% rename from .gitignore rename to expert/.gitignore diff --git a/README.md b/expert/README.md similarity index 100% rename from README.md rename to expert/README.md diff --git a/expert/bin/start b/expert/bin/start new file mode 100755 index 00000000..c946f14e --- /dev/null +++ b/expert/bin/start @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")"/.. || exit 1 + +( + cd ../engine || exit + + mix build +) + +EXPERT_ENGINE_PATH="../engine/_build/${MIX_ENV:-dev}/" mix run --no-halt -e "Application.ensure_all_started(:expert)" -- "$@" diff --git a/config/config.exs b/expert/config/config.exs similarity index 100% rename from config/config.exs rename to expert/config/config.exs diff --git a/config/dev.exs b/expert/config/dev.exs similarity index 100% rename from config/dev.exs rename to expert/config/dev.exs diff --git a/config/prod.exs b/expert/config/prod.exs similarity index 100% rename from config/prod.exs rename to expert/config/prod.exs diff --git a/config/test.exs b/expert/config/test.exs similarity index 100% rename from config/test.exs rename to expert/config/test.exs diff --git a/expert/lib/expert.ex b/expert/lib/expert.ex new file mode 100644 index 00000000..81daf6fa --- /dev/null +++ b/expert/lib/expert.ex @@ -0,0 +1,251 @@ +defmodule Expert do + use GenLSP + require Logger + require Expert.Runtime + + alias Expert.Runtime + + def start_link(args) do + {args, opts} = + Keyword.split(args, [ + :dynamic_supervisor + ]) + + GenLSP.start_link(__MODULE__, args, opts) + end + + @impl true + def init(lsp, args) do + dynamic_supervisor = Keyword.fetch!(args, :dynamic_supervisor) + + {:ok, + assign(lsp, + dynamic_supervisor: dynamic_supervisor, + exit_code: 1, + client_capabilities: nil + )} + end + + @impl true + def handle_request( + %GenLSP.Requests.Initialize{ + params: %GenLSP.Structures.InitializeParams{ + root_uri: root_uri, + workspace_folders: workspace_folders, + capabilities: caps + } + }, + lsp + ) do + System.get_env("EXPERT_ENGINE_PATH") + parent = self() + name = Path.basename(root_uri) + + working_dir = URI.parse(root_uri).path + + DynamicSupervisor.start_child( + lsp.assigns.dynamic_supervisor, + {Expert.Runtime.Supervisor, + path: Path.join(working_dir, ".expert-lsp"), + name: name, + lsp: lsp, + lsp_pid: parent, + runtime: [ + working_dir: working_dir, + uri: root_uri, + on_initialized: fn status -> + if status == :ready do + msg = {:runtime_ready, name, self()} + + Process.send(parent, msg, []) + else + send(parent, {:runtime_failed, name, status}) + end + end + ]} + ) + + {:reply, + %GenLSP.Structures.InitializeResult{ + capabilities: %GenLSP.Structures.ServerCapabilities{ + text_document_sync: %GenLSP.Structures.TextDocumentSyncOptions{ + open_close: true, + save: %GenLSP.Structures.SaveOptions{include_text: true}, + change: GenLSP.Enumerations.TextDocumentSyncKind.incremental() + }, + document_symbol_provider: true, + workspace: %{ + workspace_folders: %GenLSP.Structures.WorkspaceFoldersServerCapabilities{ + supported: true, + change_notifications: true + } + } + }, + server_info: %{name: "Expert"} + }, + assign(lsp, + root_uri: root_uri, + workspace_folders: workspace_folders, + client_capabilities: caps + )} + end + + def handle_request( + %GenLSP.Requests.TextDocumentDocumentSymbol{params: %{text_document: %{uri: uri}}}, + lsp + ) do + path = URI.parse(uri).path + doc = File.read!(path) + + lsp = + if lsp.assigns[:runtime] == nil do + receive do + {:runtime_ready, _name, runtime_pid} = msg -> + send(self(), msg) + assign(lsp, ready: true, runtime: runtime_pid) + end + else + lsp + end + + symbols = + Expert.Runtime.execute! lsp.assigns.runtime do + # we have to call the namespaced module name + XPert.DocumentSymbol.fetch(doc) + end + |> Enum.map(fn ds -> + # we also have to serialize and deserialize to send structs between nodes that + # might be namespaced + + {:ok, unified} = Schematic.unify(GenLSP.Structures.DocumentSymbol.schema(), ds) + unified + end) + + # which then will get serialized again on the way out + # we could potentially namespace our app too, but i think that + # makes our dev experience worse + + {:reply, symbols, lsp} + end + + def handle_request(_request, lsp) do + {:noreply, lsp} + end + + @impl true + def handle_notification(%GenLSP.Notifications.Initialized{}, lsp) do + Logger.info("Expert v#{version()} has initialized!") + + Logger.info("Log file located at #{Path.join(File.cwd!(), ".expert-lsp/expert.log")}") + + {:noreply, lsp} + end + + def handle_notification(_notification, lsp) do + {:noreply, lsp} + end + + def handle_info({:runtime_ready, name, runtime_pid}, lsp) do + Runtime.compile(runtime_pid) + + {:noreply, assign(lsp, ready: true, runtime: runtime_pid)} + end + + def handle_info({:compiler_result, name, result}, lsp) do + case result do + {status, diagnostics} when status in [:ok, :noop] -> + per_file = + for d <- diagnostics, reduce: Map.new() do + acc -> + diagnostic = %GenLSP.Structures.Diagnostic{ + severity: severity(d.severity), + message: IO.iodata_to_binary(d.message), + source: d.compiler_name, + range: range(d.position, Map.get(d, :span)) + } + + Map.update(acc, d.file, [diagnostic], &[diagnostic | &1]) + end + + for {file, diagnostics} <- per_file do + GenLSP.notify(lsp, %GenLSP.Notifications.TextDocumentPublishDiagnostics{ + params: %GenLSP.Structures.PublishDiagnosticsParams{ + uri: "file://#{file}", + diagnostics: diagnostics + } + }) + end + + _ -> + nil + end + + {:noreply, lsp} + end + + def version do + case :application.get_key(:expert, :vsn) do + {:ok, version} -> to_string(version) + _ -> "dev" + end + end + + defp severity(:error), do: GenLSP.Enumerations.DiagnosticSeverity.error() + defp severity(:warning), do: GenLSP.Enumerations.DiagnosticSeverity.warning() + defp severity(:info), do: GenLSP.Enumerations.DiagnosticSeverity.information() + defp severity(:hint), do: GenLSP.Enumerations.DiagnosticSeverity.hint() + + defp range({start_line, start_col, end_line, end_col}, _) do + %GenLSP.Structures.Range{ + start: %GenLSP.Structures.Position{ + line: clamp(start_line - 1), + character: start_col - 1 + }, + end: %GenLSP.Structures.Position{ + line: clamp(end_line - 1), + character: end_col - 1 + } + } + end + + defp range({startl, startc}, {endl, endc}) do + %GenLSP.Structures.Range{ + start: %GenLSP.Structures.Position{ + line: clamp(startl - 1), + character: startc - 1 + }, + end: %GenLSP.Structures.Position{ + line: clamp(endl - 1), + character: endc - 1 + } + } + end + + defp range({line, col}, nil) do + %GenLSP.Structures.Range{ + start: %GenLSP.Structures.Position{ + line: clamp(line - 1), + character: col - 1 + }, + end: %GenLSP.Structures.Position{ + line: clamp(line - 1), + character: 999 + } + } + end + + defp range(line, _) do + %GenLSP.Structures.Range{ + start: %GenLSP.Structures.Position{ + line: clamp(line - 1), + character: 0 + }, + end: %GenLSP.Structures.Position{ + line: clamp(line - 1), + character: 999 + } + } + end + + def clamp(line), do: max(line, 0) +end diff --git a/lib/expert/application.ex b/expert/lib/expert/application.ex similarity index 100% rename from lib/expert/application.ex rename to expert/lib/expert/application.ex diff --git a/lib/expert/lsp_supervisor.ex b/expert/lib/expert/lsp_supervisor.ex similarity index 64% rename from lib/expert/lsp_supervisor.ex rename to expert/lib/expert/lsp_supervisor.ex index 17911e08..95d63af9 100644 --- a/lib/expert/lsp_supervisor.ex +++ b/expert/lib/expert/lsp_supervisor.ex @@ -71,34 +71,10 @@ defmodule Expert.LSPSupervisor do System.halt(1) end - # auto_update = - # if "NEXTLS_AUTO_UPDATE" |> System.get_env("false") |> String.to_existing_atom() do - # [ - # binpath: - # System.get_env( - # "NEXTLS_BINPATH", - # Path.expand("~/.cache/elixir-tools/nextls/bin/nextls") - # ), - # api_host: System.get_env("NEXTLS_GITHUB_API", "https://api.github.com"), - # github_host: System.get_env("NEXTLS_GITHUB", "https://github.com"), - # current_version: Version.parse!(NextLS.version()) - # ] - # else - # false - # end - children = [ - {GenLSP.Buffer, [name: NextLS.Buffer] ++ buffer_opts}, - { - Expert, - # auto_update: auto_update, - buffer: NextLS.Buffer - # cache: :diagnostic_cache, - # task_supervisor: NextLS.TaskSupervisor, - # runtime_task_supervisor: :runtime_task_supervisor, - # dynamic_supervisor: NextLS.DynamicSupervisor, - # registry: NextLS.Registry} - } + {DynamicSupervisor, name: Expert.DynamicSupervisor}, + {GenLSP.Buffer, [name: Expert.Buffer] ++ buffer_opts}, + {Expert, buffer: Expert.Buffer, dynamic_supervisor: Expert.DynamicSupervisor} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/expert/lib/expert/release.ex b/expert/lib/expert/release.ex new file mode 100644 index 00000000..2588279f --- /dev/null +++ b/expert/lib/expert/release.ex @@ -0,0 +1,27 @@ +defmodule Expert.Release do + def assemble(release) do + engine_path = Path.expand("../../../engine", __DIR__) + + {_, 0} = + System.cmd("mix", ["build"], + cd: engine_path, + env: [ + {"MIX_ENV", to_string(Mix.env())} + ] + ) + + source = Path.join([engine_path, "_build/prod"]) + + dest = + Path.join([ + release.path, + "lib", + "#{release.name}-#{release.version}", + "priv" + ]) + + File.cp_r!(source, dest) + + release + end +end diff --git a/expert/lib/expert/runtime.ex b/expert/lib/expert/runtime.ex new file mode 100644 index 00000000..674970fb --- /dev/null +++ b/expert/lib/expert/runtime.ex @@ -0,0 +1,328 @@ +defmodule Expert.Runtime do + @moduledoc false + use GenServer + + @env Mix.env() + defguardp is_ready(state) when is_map_key(state, :node) + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + + @type mod_fun_arg :: {atom(), atom(), list()} + + @spec call(pid(), mod_fun_arg()) :: any() + def call(server, mfa) do + GenServer.call(server, {:call, mfa}, :infinity) + end + + @spec expand(pid(), Macro.t(), String.t()) :: any() + def expand(server, ast, file) do + GenServer.call(server, {:expand, ast, file}, :infinity) + end + + @spec ready?(pid()) :: boolean() + def ready?(server), do: GenServer.call(server, :ready?) + + @spec await(pid(), non_neg_integer()) :: :ok | :timeout + def await(server, count \\ 50) + + def await(_server, 0) do + :timeout + end + + def await(server, count) do + with {:alive, true} <- {:alive, Process.alive?(server)}, + true <- ready?(server) do + :ok + else + {:alive, false} -> + :timeout + + _ -> + Process.sleep(500) + await(server, count - 1) + end + end + + @spec compile(pid(), Keyword.t()) :: any() + def compile(server, opts \\ []) do + GenServer.call(server, {:compile, opts}, :infinity) + end + + def boot(supervisor, opts) do + DynamicSupervisor.start_child(supervisor, {Expert.Runtime.Supervisor, opts}) + end + + def stop(supervisor, pid) do + DynamicSupervisor.terminate_child(supervisor, pid) + end + + defmacro execute!(runtime, block) do + quote do + {:ok, result} = Expert.Runtime.execute(unquote_splicing([runtime, block])) + result + end + end + + defmacro execute(runtime, do: block) do + exprs = + case block do + {:__block__, _, exprs} -> exprs + expr -> [expr] + end + + for expr <- exprs, reduce: quote(do: :ok) do + ast -> + mfa = + case expr do + {{:., _, [mod, func]}, _, args} -> + [mod, func, args] + + {_func, _, _args} -> + raise "#{Macro.to_string(__MODULE__)}.execute/2 cannot be called with local functions" + end + + quote do + unquote(ast) + Expert.Runtime.call(unquote(runtime), {unquote_splicing(mfa)}) + end + end + end + + @impl GenServer + def init(opts) do + sname = "expert-runtime-#{System.system_time()}" + name = Keyword.fetch!(opts, :name) + working_dir = Keyword.fetch!(opts, :working_dir) + lsp_pid = Keyword.fetch!(opts, :lsp_pid) + # uri = Keyword.fetch!(opts, :uri) + parent = Keyword.fetch!(opts, :parent) + on_initialized = Keyword.fetch!(opts, :on_initialized) + + elixir_exe = System.find_executable("elixir") + + pid = + cond do + is_pid(parent) -> parent + is_atom(parent) -> Process.whereis(parent) + end + + parent = + pid + |> :erlang.term_to_binary() + |> Base.encode64() + |> String.to_charlist() + + bindir = System.get_env("BINDIR") + path = System.get_env("PATH") + path_minus_bindir = String.replace(path, bindir <> ":", "") + + path_minus_bindir2 = + path_minus_bindir |> String.split(":") |> List.delete(bindir) |> Enum.join(":") + + new_path = elixir_exe <> ":" <> path_minus_bindir2 + + case :code.priv_dir(:expert) do + dir when is_list(dir) -> + exe = + dir + |> Path.join("cmd") + |> Path.absname() + + engine_path = + System.get_env("EXPERT_ENGINE_PATH", to_string(dir)) |> Path.expand() + + env = + [ + {~c"LSP", ~c"expert"}, + {~c"EXPERT_PARENT_PID", parent}, + {~c"MIX_BUILD_ROOT", ~c".expert-lsp/_build"}, + {~c"ROOTDIR", false}, + {~c"BINDIR", false}, + {~c"RELEASE_ROOT", false}, + {~c"RELEASE_SYS_CONFIG", false}, + {~c"PATH", String.to_charlist(new_path)} + ] + + consolidated = + Path.wildcard(Path.join(engine_path, "lib/*/{consolidated}")) + |> Enum.flat_map(fn ep -> ["-pa", ep] end) + + rest = + Path.wildcard(Path.join(engine_path, "lib/*/{ebin}")) + |> Enum.flat_map(fn ep -> ["-pa", ep] end) + + engine_path_args = rest ++ consolidated + + args = + [elixir_exe] ++ + if @env == :test do + ["--erl", "-kernel prevent_overlapping_partitions false"] + else + [] + end ++ + engine_path_args ++ + [ + "--no-halt", + "--sname", + sname, + "--cookie", + Node.get_cookie(), + "-S", + "mix", + "loadpaths", + "--no-compile" + ] + + port = + Port.open( + {:spawn_executable, exe}, + [ + :use_stdio, + :stderr_to_stdout, + :binary, + :stream, + cd: working_dir, + env: env, + args: args + ] + ) + + Port.monitor(port) + + me = self() + + Task.start_link(fn -> + {:ok, host} = :inet.gethostname() + node = :"#{sname}@#{host}" + + case connect(node, port, 120) do + true -> + {:ok, _} = :rpc.call(node, Application, :ensure_all_started, [:xp_engine]) + + send(me, {:node, node}) + + error -> + send(me, {:cancel, error}) + end + end) + + {:ok, + %{ + name: name, + working_dir: working_dir, + compiler_refs: %{}, + port: port, + lsp_pid: lsp_pid, + parent: parent, + errors: nil, + on_initialized: on_initialized + }} + + _ -> + {:stop, :failed_to_boot} + end + end + + @impl GenServer + def handle_call(:ready?, _from, state) when is_ready(state) do + {:reply, true, state} + end + + def handle_call(:ready?, _from, state) do + {:reply, false, state} + end + + def handle_call(_, _from, state) when not is_ready(state) do + {:reply, {:error, :not_ready}, state} + end + + def handle_call({:call, {m, f, a}}, _from, %{node: node} = state) do + reply = :rpc.call(node, m, f, a) + {:reply, {:ok, reply}, state} + end + + def handle_call({:compile, opts}, _from, %{node: node} = state) do + opts = + opts + |> Keyword.put_new(:working_dir, state.working_dir) + |> Keyword.put(:from, self()) + + with {:badrpc, _error} <- + :rpc.call(node, XPert.Worker, :enqueue_compiler, [opts]) do + :error + end + + {:reply, :ok, state} + end + + @impl GenServer + # NOTE: these two callbacks are basically to forward the messages from the runtime to the + # LSP process so that progress messages can be dispatched + def handle_info({:compiler_result, result}, state) do + # we add the runtime name into the message + send(state.lsp_pid, {:compiler_result, state.name, result}) + + {:noreply, state} + end + + def handle_info({:DOWN, _, :port, port, _}, %{port: port} = state) do + unless is_ready(state) do + state.on_initialized.({:error, :portdown}) + end + + {:noreply, Map.delete(state, :node)} + end + + def handle_info({:cancel, error}, state) do + state.on_initialized.({:error, error}) + {:noreply, Map.delete(state, :node)} + end + + def handle_info({:node, node}, state) do + Node.monitor(node, true) + state.on_initialized.(:ready) + {:noreply, Map.put(state, :node, node)} + end + + def handle_info({:nodedown, node}, %{node: node} = state) do + {:stop, {:shutdown, :nodedown}, state} + end + + def handle_info( + {port, {:data, "** (Mix) Can't continue due to errors on dependencies" <> _ = _data}}, + %{port: port} = state + ) do + Port.close(port) + state.on_initialized.({:error, :deps}) + {:stop, {:shutdown, :unchecked_dependencies}, state} + end + + def handle_info({port, {:data, "Unchecked dependencies" <> _ = _data}}, %{port: port} = state) do + Port.close(port) + state.on_initialized.({:error, :deps}) + {:stop, {:shutdown, :unchecked_dependencies}, state} + end + + def handle_info({port, {:data, _data}}, %{port: port} = state) do + {:noreply, state} + end + + def handle_info({port, _other}, %{port: port} = state) do + {:noreply, state} + end + + defp connect(_node, _port, 0) do + false + end + + defp connect(node, port, attempts) do + if Node.connect(node) in [false, :ignored] do + Process.sleep(1000) + connect(node, port, attempts - 1) + else + true + end + end +end diff --git a/expert/lib/expert/runtime/supervisor.ex b/expert/lib/expert/runtime/supervisor.ex new file mode 100644 index 00000000..5c6abcfe --- /dev/null +++ b/expert/lib/expert/runtime/supervisor.ex @@ -0,0 +1,26 @@ +defmodule Expert.Runtime.Supervisor do + @moduledoc false + + use Supervisor + + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg) + end + + @impl true + def init(init_arg) do + name = init_arg[:name] + lsp_pid = init_arg[:lsp_pid] + hidden_folder = init_arg[:path] + File.mkdir_p!(hidden_folder) + File.write!(Path.join(hidden_folder, ".gitignore"), "*\n") + + children = [ + {Expert.Runtime, + init_arg[:runtime] ++ + [name: name, parent: lsp_pid, lsp_pid: lsp_pid]} + ] + + Supervisor.init(children, strategy: :one_for_one) + end +end diff --git a/mix.exs b/expert/mix.exs similarity index 77% rename from mix.exs rename to expert/mix.exs index 5a495b48..c57bb7c8 100644 --- a/mix.exs +++ b/expert/mix.exs @@ -26,7 +26,7 @@ defmodule Expert.MixProject do [ plain: [], expert: [ - steps: [:assemble, &Burrito.wrap/1], + steps: [:assemble, &Expert.Release.assemble/1, &Burrito.wrap/1], burrito: [ targets: [ darwin_arm64: [os: :darwin, cpu: :aarch64], @@ -46,8 +46,11 @@ defmodule Expert.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:gen_lsp, "~> 0.10"}, - {:burrito, "~> 1.0", only: [:dev, :prod]} + {:gen_lsp, + github: "elixir-tools/gen_lsp", branch: "change-schematic-function", override: true}, + # {:gen_lsp, "~> 0.10"}, + {:burrito, "~> 1.0", only: [:dev, :prod]}, + {:namespace, path: "../namespace", only: [:dev]} ] end end diff --git a/mix.lock b/expert/mix.lock similarity index 88% rename from mix.lock rename to expert/mix.lock index fcb54acd..ceab83ed 100644 --- a/mix.lock +++ b/expert/mix.lock @@ -1,7 +1,7 @@ %{ "burrito": {:hex, :burrito, "1.2.0", "88f973469edcb96bd984498fb639d3fc4dbf01b52baab072b40229f03a396789", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:req, ">= 0.4.0", [hex: :req, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.2.0 or ~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "7e22158023c6558de615795ab135d27f0cbd9a0602834e3e474fe41b448afba9"}, "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, - "gen_lsp": {:hex, :gen_lsp, "0.10.0", "f6da076b5ccedf937d17aa9743635a2c3d0f31265c853e58b02ab84d71852270", [:mix], [{:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:schematic, "~> 0.2.1", [hex: :schematic, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "768f8f7b5c5e218fb36dcebd30dcd6275b61ca77052c98c3c4c0375158392c4a"}, + "gen_lsp": {:git, "https://github.com/elixir-tools/gen_lsp.git", "f63f284289ef61b678ab6bd2cbcf3a6ba3d9fcdd", [branch: "change-schematic-function"]}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, diff --git a/expert/priv/cmd b/expert/priv/cmd new file mode 100755 index 00000000..6121c23a --- /dev/null +++ b/expert/priv/cmd @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Start the program in the background +exec "$@" & +pid1=$! + +# Silence warnings from here on +exec >/dev/null 2>&1 + +# Read from stdin in the background and +# kill running program when stdin closes +exec 0<&0 $( + while read; do :; done + kill -KILL $pid1 +) & +pid2=$! + +# Clean up +wait $pid1 +ret=$? +kill -KILL $pid2 +exit $ret diff --git a/test/expert_test.exs b/expert/test/expert_test.exs similarity index 100% rename from test/expert_test.exs rename to expert/test/expert_test.exs diff --git a/expert/test/test_helper.exs b/expert/test/test_helper.exs new file mode 100644 index 00000000..869559e7 --- /dev/null +++ b/expert/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/justfile b/justfile index 5cef9e9f..b6e6d95d 100644 --- a/justfile +++ b/justfile @@ -1,32 +1,37 @@ -default: deps compile build-local - -choose: - just --choose - -deps: +deps project: + #!/usr/bin/env bash + cd {{project}} mix deps.get -compile: +compile project: + #!/usr/bin/env bash + cd {{project}} mix compile +build: + #!/usr/bin/env bash + cd engine + mix build + start: + #!/usr/bin/env bash + cd expert bin/start --port 9000 -test: +test project: + #!/usr/bin/env bash + cd {{project}} mix test -format: - mix format - -lint: +format project: #!/usr/bin/env bash - set -euxo pipefail - - mix format --check-formatted + cd {{project}} + mix format [unix] build-local: #!/usr/bin/env bash + cd expert case "{{os()}}-{{arch()}}" in "linux-arm" | "linux-aarch64") target=linux_arm64;; @@ -49,10 +54,9 @@ build-local: EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release build-all: + cd expert EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release build-plain: + cd expert MIX_ENV=prod mix release plain - -bump-spitfire: - mix deps.update spitfire diff --git a/lib/expert.ex b/lib/expert.ex deleted file mode 100644 index 69a3289f..00000000 --- a/lib/expert.ex +++ /dev/null @@ -1,83 +0,0 @@ -defmodule Expert do - use GenLSP - require Logger - - def start_link(opts) do - GenLSP.start_link(__MODULE__, [], opts) - end - - @impl true - def init(lsp, _args) do - {:ok, - assign(lsp, - exit_code: 1, - client_capabilities: nil - )} - end - - @impl true - def handle_request( - %GenLSP.Requests.Initialize{ - params: %GenLSP.Structures.InitializeParams{ - root_uri: root_uri, - workspace_folders: workspace_folders, - capabilities: caps - } - }, - lsp - ) do - workspace_folders = - if caps.workspace.workspace_folders do - workspace_folders - else - [%{name: Path.basename(root_uri), uri: root_uri}] - end - - {:reply, - %GenLSP.Structures.InitializeResult{ - capabilities: %GenLSP.Structures.ServerCapabilities{ - text_document_sync: %GenLSP.Structures.TextDocumentSyncOptions{ - open_close: true, - save: %GenLSP.Structures.SaveOptions{include_text: true}, - change: GenLSP.Enumerations.TextDocumentSyncKind.incremental() - }, - workspace: %{ - workspace_folders: %GenLSP.Structures.WorkspaceFoldersServerCapabilities{ - supported: true, - change_notifications: true - } - } - }, - server_info: %{name: "Expert"} - }, - assign(lsp, - root_uri: root_uri, - workspace_folders: workspace_folders, - client_capabilities: caps - )} - end - - def handle_request(_request, lsp) do - {:noreply, lsp} - end - - @impl true - def handle_notification(%GenLSP.Notifications.Initialized{}, lsp) do - Logger.info("Expert v#{version()} has initialized!") - - Logger.info("Log file located at #{Path.join(File.cwd!(), ".expert-lsp/expert.log")}") - - {:noreply, lsp} - end - - def handle_notification(_notification, lsp) do - {:noreply, lsp} - end - - def version do - case :application.get_key(:expert, :vsn) do - {:ok, version} -> to_string(version) - _ -> "dev" - end - end -end diff --git a/namespace/.formatter.exs b/namespace/.formatter.exs new file mode 100644 index 00000000..d2cda26e --- /dev/null +++ b/namespace/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/namespace/.gitignore b/namespace/.gitignore new file mode 100644 index 00000000..cecd9d45 --- /dev/null +++ b/namespace/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +namespace-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/namespace/README.md b/namespace/README.md new file mode 100644 index 00000000..b82e3e82 --- /dev/null +++ b/namespace/README.md @@ -0,0 +1,21 @@ +# Namespace + +**TODO: Add description** + +## Installation + +If [available in Hex](https://hex.pm/docs/publish), the package can be installed +by adding `namespace` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [ + {:namespace, "~> 0.1.0"} + ] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at . + diff --git a/namespace/lib/mix/tasks/namespace.ex b/namespace/lib/mix/tasks/namespace.ex new file mode 100644 index 00000000..27933e9e --- /dev/null +++ b/namespace/lib/mix/tasks/namespace.ex @@ -0,0 +1,149 @@ +defmodule Mix.Tasks.Namespace do + @moduledoc """ + This task is used after a release is assembled, and investigates the remote_control + app for its dependencies, at which point it applies transformers to various parts of the + app. + + Transformers take a path, find their relevant files and apply transforms to them. For example, + the Beams transformer will find any instances of modules in .beam files, and will apply namepaces + to them if the module is one of the modules defined in a dependency. + + This task takes a single argument, which is the full path to the release. + """ + alias Mix.Tasks.Namespace.Transform + use Mix.Task + + @dev_deps [:namespace] + + # These app names and root modules are strings to avoid them being namespaced + # by this task. Plugin discovery uses this task, which happens after + # namespacing. + @extra_apps %{ + "engine" => "Engine" + } + + defp deps_apps() do + Mix.Project.deps_apps() + |> Kernel.--(@dev_deps) + |> Enum.map(&to_string/1) + end + + require Logger + + def run([base_directory]) do + Process.put(:deps_apps, deps_apps()) + Transform.Apps.apply_to_all(base_directory) + Transform.Beams.apply_to_all(base_directory) + # consolidated + + # Transform.Beams.apply(base_directory) + Transform.Scripts.apply_to_all(base_directory) + # The boot file transform just turns script files into boot files + # so it must come after the script file transform + Transform.Boots.apply_to_all(base_directory) + Transform.AppDirectories.apply_to_all(base_directory) + end + + def app_names() do + Map.keys(app_to_root_modules()) + end + + def root_modules() do + app_to_root_modules() + |> Map.values() + |> List.flatten() + end + + def app_to_root_modules() do + case :persistent_term.get(__MODULE__, :not_loaded) do + :not_loaded -> + init() + + term -> + term + end + end + + defp register_mappings(app_to_root_modules) do + :persistent_term.put(__MODULE__, app_to_root_modules) + app_to_root_modules + end + + defp root_modules_for_apps(deps_apps) do + deps_apps + |> Enum.map(fn app_name -> + all_modules = app_modules(app_name) + + case Enum.filter(all_modules, fn module -> length(safe_split_module(module)) == 1 end) do + [] -> {app_name, [Expert]} + root_modules -> {app_name, root_modules} + end + end) + |> Enum.reject(&is_nil/1) + |> Map.new() + end + + defp app_modules(dep_app) do + Application.ensure_loaded(dep_app) + + case :application.get_key(dep_app, :modules) do + {:ok, modules} -> + modules + + _ -> + [Expert] + end + end + + defp safe_split_module(module) do + case safe_split(module) do + {:elixir, segments} -> segments + {:erlang, _} -> [] + end + end + + defp extra_apps do + Map.new(@extra_apps, fn {k, v} -> + root_module = + v + |> List.wrap() + |> Module.concat() + + {String.to_atom(k), [root_module]} + end) + end + + defp init() do + Process.get(:deps_apps) + |> Enum.map(&String.to_atom/1) + |> root_modules_for_apps() + |> Map.merge(extra_apps()) + |> register_mappings() + end + + def safe_split(module, opts \\ []) + + def safe_split(module, opts) when is_atom(module) do + string_name = Atom.to_string(module) + + {type, split_module} = + case String.split(string_name, ".") do + ["Elixir" | rest] -> + {:elixir, rest} + + [_erlang_module] = module -> + {:erlang, module} + end + + split_module = + case Keyword.get(opts, :as, :binaries) do + :binaries -> + split_module + + :atoms -> + Enum.map(split_module, &String.to_atom/1) + end + + {type, split_module} + end +end diff --git a/namespace/lib/mix/tasks/namespace/abstract.ex b/namespace/lib/mix/tasks/namespace/abstract.ex new file mode 100644 index 00000000..0f7d2d67 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/abstract.ex @@ -0,0 +1,297 @@ +defmodule Mix.Tasks.Namespace.Abstract do + @moduledoc """ + Transformations from erlang abstract syntax + + The abstract syntax is rather tersely defined here: + https://www.erlang.org/doc/apps/erts/absform.html + """ + + alias Mix.Tasks.Namespace + + def rewrite(abstract_format) when is_list(abstract_format) do + Enum.map(abstract_format, &rewrite/1) + end + + def rewrite(abstract_format) do + do_rewrite(abstract_format) + end + + # 8.1 Module Declarations and Forms + + defp do_rewrite({:attribute, anno, :export, exported_functions}) do + {:attribute, anno, :export, exported_functions} + end + + defp do_rewrite({:attribute, anno, :behaviour, module}) do + {:attribute, anno, :behaviour, rewrite_module(module)} + end + + defp do_rewrite({:attribute, anno, :import, {module, funs}}) do + {:attribute, anno, :import, {rewrite_module(module), rewrite(funs)}} + end + + defp do_rewrite({:attribute, anno, :module, mod}) do + {:attribute, anno, :module, rewrite_module(mod)} + end + + defp do_rewrite({:attribute, anno, :__impl__, attrs}) do + {:attribute, anno, :__impl__, rewrite(attrs)} + end + + defp do_rewrite({:function, anno, name, arity, clauses} = full) do + {:function, anno, name, arity, rewrite(clauses)} + end + + defp do_rewrite({:attribute, anno, spec, {{name, arity}, spec_clauses}}) do + {:attribute, anno, rewrite(spec), {{name, arity}, rewrite(spec_clauses)}} + end + + defp do_rewrite({:attribute, anno, :spec, {{mod, name, arity}, clauses}}) do + {:attribute, anno, :spec, {{rewrite(mod), name, arity}, rewrite(clauses)}} + end + + defp do_rewrite({:attribute, anno, :record, {name, fields}}) do + {:attribute, anno, :record, {rewrite_module(name), rewrite(fields)}} + end + + defp do_rewrite({:attribute, anno, type, {name, type_rep, clauses}}) do + {:attribute, anno, type, {name, rewrite(type_rep), rewrite(clauses)}} + end + + defp do_rewrite({:for, target}) do + # Protocol implementation + {:for, rewrite_module(target)} + end + + defp do_rewrite({:protocol, protocol}) do + {:protocol, rewrite_module(protocol)} + end + + # Record Fields + + defp do_rewrite({:record_field, anno, repr}) do + {:record_field, anno, rewrite(repr)} + end + + defp do_rewrite({:record_field, anno, repr_1, repr_2}) do + {:record_field, anno, rewrite(repr_1), rewrite(repr_2)} + end + + defp do_rewrite({:typed_record_field, {:record_field, anno, repr_1}, repr_2}) do + {:typed_record_field, {:record_field, anno, rewrite(repr_1)}, rewrite(repr_2)} + end + + defp do_rewrite({:typed_record_field, {:record_field, anno, repr_a, repr_e}, repr_t}) do + {:typed_record_field, {:record_field, anno, rewrite(repr_a), rewrite(repr_e)}, + rewrite(repr_t)} + end + + # Representation of Parse Errors and End-of-File Omitted; not necessary + # 8.2 Atomic Literals + + # only rewrite atoms, since they might be modules + defp do_rewrite({:atom, anno, literal}) do + {:atom, anno, rewrite_module(literal)} + end + + # 8.3 Patterns + # ignore bitstraings, they can't contain modules + + defp do_rewrite({:match, anno, lhs, rhs}) do + {:match, anno, rewrite(lhs), rewrite(rhs)} + end + + defp do_rewrite({:cons, anno, head, tail}) do + {:cons, anno, rewrite(head), rewrite(tail)} + end + + defp do_rewrite({:map, anno, matches}) do + {:map, anno, rewrite(matches)} + end + + defp do_rewrite({:op, anno, op, lhs, rhs}) do + {:op, anno, op, rewrite(lhs), rewrite(rhs)} + end + + defp do_rewrite({:op, anno, op, pattern}) do + {:op, anno, op, rewrite(pattern)} + end + + defp do_rewrite({:tuple, anno, patterns}) do + {:tuple, anno, rewrite(patterns)} + end + + defp do_rewrite({:var, anno, atom}) do + {:var, anno, rewrite_module(atom)} + end + + # 8.4 Expressions + + defp do_rewrite({:bc, anno, rep_e0, qualifiers}) do + {:bc, anno, rewrite(rep_e0), rewrite(qualifiers)} + end + + defp do_rewrite({:bin, anno, bin_elements}) do + {:bin, anno, rewrite(bin_elements)} + end + + defp do_rewrite({:bin_element, anno, elem, size, type}) do + {:bin_element, anno, rewrite(elem), size, type} + end + + defp do_rewrite({:block, anno, body}) do + {:block, anno, rewrite(body)} + end + + defp do_rewrite({:case, anno, expression, clauses}) do + {:case, anno, rewrite(expression), rewrite(clauses)} + end + + defp do_rewrite({:catch, anno, expression}) do + {:catch, anno, rewrite(expression)} + end + + defp do_rewrite({:fun, anno, {:function, name, arity}} = full) do + {:fun, anno, {:function, rewrite(name), arity}} + end + + defp do_rewrite({:fun, anno, {:function, module, name, arity}} = full) do + {:fun, anno, {:function, rewrite(module), rewrite(name), arity}} + end + + defp do_rewrite({:fun, anno, {:clauses, clauses}}) do + {:fun, anno, {:clauses, rewrite(clauses)}} + end + + defp do_rewrite({:named_fun, anno, name, clauses} = full) do + {:named_fun, anno, rewrite(name), rewrite(clauses)} + end + + defp do_rewrite({:call, anno, {:remote, remote_anno, module, fn_name}, args} = full) do + {:call, anno, {:remote, remote_anno, rewrite(module), fn_name}, rewrite(args)} + end + + defp do_rewrite({:call, anno, name, args}) do + {:call, anno, rewrite(name), rewrite(args)} + end + + defp do_rewrite({:if, anno, clauses}) do + {:if, anno, rewrite(clauses)} + end + + defp do_rewrite({:lc, anno, expression, qualifiers}) do + {:lc, anno, rewrite(expression), rewrite(qualifiers)} + end + + defp do_rewrite({:map, anno, expression, clauses}) do + {:map, anno, rewrite(expression), rewrite(clauses)} + end + + defp do_rewrite({:maybe_match, anno, lhs, rhs}) do + {:maybe_match, anno, rewrite(lhs), rewrite(rhs)} + end + + defp do_rewrite({:maybe, anno, body}) do + {:maybe, anno, rewrite(body)} + end + + defp do_rewrite({:maybe, anno, maybe_body, {:else, anno, else_clauses}}) do + {:maybe, anno, rewrite(maybe_body), {:else, anno, rewrite(else_clauses)}} + end + + defp do_rewrite({:receive, anno, clauses}) do + {:receive, anno, rewrite(clauses)} + end + + defp do_rewrite({:receive, anno, cases, expression, body}) do + {:receive, anno, rewrite(cases), rewrite(expression), rewrite(body)} + end + + defp do_rewrite({:record, anno, name, fields}) do + {:record, anno, rewrite_module(name), rewrite(fields)} + end + + defp do_rewrite({:record_field, anno, record_name, field_name, record_field}) do + {:record_field, anno, rewrite_module(record_name), field_name, record_field} + end + + defp do_rewrite({:try, anno, body, case_clauses, catch_clauses}) do + {:try, anno, rewrite(body), rewrite(case_clauses), rewrite(catch_clauses)} + end + + defp do_rewrite({:try, anno, body, case_clauses, catch_clauses, after_clauses}) do + {:try, anno, rewrite(body), rewrite(case_clauses), rewrite(catch_clauses), + rewrite(after_clauses)} + end + + # Qualifiers + + defp do_rewrite({:generate, anno, lhs, rhs}) do + {:generate, anno, rewrite(lhs), rewrite(rhs)} + end + + defp do_rewrite({:b_generate, anno, lhs, rhs}) do + {:b_generate, anno, rewrite(lhs), rewrite(rhs)} + end + + # Associations + + defp do_rewrite({:map_field_assoc, anno, key, value}) do + {:map_field_assoc, anno, rewrite(key), rewrite(value)} + end + + defp do_rewrite({:map_field_exact, anno, key, value}) do + {:map_field_exact, anno, rewrite(key), rewrite(value)} + end + + # 8.5 Clauses + + defp do_rewrite({:clause, anno, lhs, guards, rhs}) do + {:clause, anno, rewrite(lhs), rewrite(guards), rewrite(rhs)} + end + + # 8.6 Guards + # Guards seem covered by above clauses + + # 8.7 Types + defp do_rewrite({:ann_type, anno, clauses}) do + {:ann_type, anno, rewrite(clauses)} + end + + defp do_rewrite({:type, anno, :fun, [{:type, type_anno, :any}, type]}) do + {:type, anno, :fun, [{:type, type_anno, :any}, rewrite(type)]} + end + + defp do_rewrite({:type, anno, :map, key_values}) do + {:type, anno, :map, rewrite(key_values)} + end + + defp do_rewrite({:type, anno, predefined_type, expressions}) do + {:type, anno, rewrite(predefined_type), rewrite(expressions)} + end + + defp do_rewrite({:remote_type, anno, [module, name, expressions]}) do + {:remote_type, anno, [rewrite_module(module), name, rewrite(expressions)]} + end + + defp do_rewrite({:user_type, anno, name, types}) do + {:user_type, anno, rewrite_module(name), rewrite(types)} + end + + # Catch all + defp do_rewrite(other) do + other + end + + defp rewrite_module({:atom, sequence, literal}) do + {:atom, sequence, rewrite_module(literal)} + end + + defp rewrite_module({:var, anno, name}) do + {:var, anno, rewrite_module(name)} + end + + defp rewrite_module(module) do + Namespace.Module.apply(module) + end +end diff --git a/namespace/lib/mix/tasks/namespace/code.ex b/namespace/lib/mix/tasks/namespace/code.ex new file mode 100644 index 00000000..60523a3d --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/code.ex @@ -0,0 +1,5 @@ +defmodule Mix.Tasks.Namespace.Code do + def compile(forms) do + :compile.forms(forms, [:return_errors, :debug_info]) + end +end diff --git a/namespace/lib/mix/tasks/namespace/module.ex b/namespace/lib/mix/tasks/namespace/module.ex new file mode 100644 index 00000000..cf7b5361 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/module.ex @@ -0,0 +1,83 @@ +defmodule Mix.Tasks.Namespace.Module do + alias Mix.Tasks.Namespace + + @namespace_prefix "XP" + + def apply(module_name) do + cond do + prefixed?(module_name) -> + module_name + + module_name in Namespace.app_names() -> + :"xp_#{module_name}" + + true -> + module_name + |> Atom.to_string() + |> apply_namespace() + end + end + + def prefixed?(module) when is_atom(module) do + module + |> Atom.to_string() + |> prefixed?() + end + + def prefixed?("Elixir." <> rest), + do: prefixed?(rest) + + def prefixed?(@namespace_prefix <> _), + do: true + + def prefixed?("xp_" <> _), + do: true + + def prefixed?([?x, ?p, ?_ | _]), do: true + def prefixed?([?E, ?l, ?i, ?x, ?i, ?r, ?., ?X, ?P | _]), do: true + def prefixed?([?X, ?P | _]), do: true + + def prefixed?(_), + do: false + + defp apply_namespace("Elixir." <> rest) do + Namespace.root_modules() + |> Enum.map(fn module -> module |> Module.split() |> List.first() end) + |> Enum.reduce_while(rest, fn root_module, module -> + if has_root_module?(root_module, module) do + namespaced_module = + module + |> String.replace(root_module, namespace(root_module), global: false) + |> String.to_atom() + + {:halt, namespaced_module} + else + {:cont, module} + end + end) + |> List.wrap() + |> Module.concat() + end + + defp apply_namespace(erlang_module) do + String.to_atom(erlang_module) + end + + defp has_root_module?(root_module, root_module), do: true + + defp has_root_module?(root_module, candidate) do + String.contains?(candidate, append_trailing_period(root_module)) + end + + defp namespace("Engine") do + "#{@namespace_prefix}ert" + end + + defp namespace(orig) do + @namespace_prefix <> orig + end + + defp append_trailing_period(str) do + str <> "." + end +end diff --git a/namespace/lib/mix/tasks/namespace/path.ex b/namespace/lib/mix/tasks/namespace/path.ex new file mode 100644 index 00000000..aaf1778f --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/path.ex @@ -0,0 +1,29 @@ +defmodule Mix.Tasks.Namespace.Path do + alias Mix.Tasks.Namespace + + def apply(path) when is_list(path) do + path + |> List.to_string() + |> apply() + |> String.to_charlist() + end + + def apply(path) when is_binary(path) do + path + |> Path.split() + |> Enum.map(&replace_namespaced_apps/1) + |> Path.join() + end + + defp replace_namespaced_apps(path_component) do + Enum.reduce(Namespace.app_names(), path_component, fn app_name, path -> + if path == Atom.to_string(app_name) do + app_name + |> Namespace.Module.apply() + |> Atom.to_string() + else + path + end + end) + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/app_directories.ex b/namespace/lib/mix/tasks/namespace/transform/app_directories.ex new file mode 100644 index 00000000..c8f844d3 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/app_directories.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Namespace.Transform.AppDirectories do + alias Mix.Tasks.Namespace + + def apply_to_all(base_directory) do + base_directory + |> find_app_directories() + |> Enum.each(&apply/1) + end + + def apply(app_path) do + namespaced_app_path = Namespace.Path.apply(app_path) + + with {:ok, _} <- File.rm_rf(namespaced_app_path) do + File.rename!(app_path, namespaced_app_path) + end + end + + defp find_app_directories(base_directory) do + app_globs = Enum.join(Namespace.app_names(), "*,") + + [base_directory, "lib", "{" <> app_globs <> "*}"] + |> Path.join() + |> Path.wildcard() + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/apps.ex b/namespace/lib/mix/tasks/namespace/transform/apps.ex new file mode 100644 index 00000000..a3c1e938 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/apps.ex @@ -0,0 +1,79 @@ +defmodule Mix.Tasks.Namespace.Transform.Apps do + @moduledoc """ + Applies namespacing to all modules defined in .app files + """ + alias Mix.Tasks.Namespace + alias Mix.Tasks.Namespace.Transform + + def apply_to_all(base_directory) do + base_directory + |> find_app_files() + |> tap(fn app_files -> + Mix.Shell.IO.info("Rewriting #{length(app_files)} app files") + end) + |> Enum.each(&apply/1) + end + + def apply(file_path) do + with {:ok, app_definition} <- Transform.Erlang.path_to_term(file_path), + {:ok, converted} <- convert(app_definition), + :ok <- File.write(file_path, converted) do + app_name = + file_path + |> Path.basename() + |> Path.rootname() + |> String.to_atom() + + namespaced_app_name = Namespace.Module.apply(app_name) + new_filename = "#{namespaced_app_name}.app" + + new_file_path = + file_path + |> Path.dirname() + |> Path.join(new_filename) + + File.rename!(file_path, new_file_path) + end + end + + defp find_app_files(base_directory) do + app_files_glob = Enum.join(Namespace.app_names(), ",") + + [base_directory, "**", "{#{app_files_glob}}.app"] + |> Path.join() + |> Path.wildcard() + end + + defp convert(app_definition) do + erlang_terms = + app_definition + |> visit() + |> Transform.Erlang.term_to_string() + + {:ok, erlang_terms} + end + + defp visit({:application, app_name, keys}) do + {:application, Namespace.Module.apply(app_name), Enum.map(keys, &visit/1)} + end + + defp visit({:applications, app_list}) do + {:applications, Enum.map(app_list, &Namespace.Module.apply/1)} + end + + defp visit({:modules, module_list}) do + {:modules, Enum.map(module_list, &Namespace.Module.apply/1)} + end + + defp visit({:description, desc}) do + {:description, desc ++ ~c" namespaced by expert."} + end + + defp visit({:mod, {module_name, args}}) do + {:mod, {Namespace.Module.apply(module_name), args}} + end + + defp visit(key_value) do + key_value + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/beams.ex b/namespace/lib/mix/tasks/namespace/transform/beams.ex new file mode 100644 index 00000000..c35d9047 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/beams.ex @@ -0,0 +1,117 @@ +defmodule Mix.Tasks.Namespace.Transform.Beams do + @moduledoc """ + A transformer that finds and replaces any instance of a module in a .beam file + """ + + alias Mix.Tasks.Namespace + alias Mix.Tasks.Namespace.Abstract + alias Mix.Tasks.Namespace.Code + + def apply_to_all(base_directory) do + Mix.Shell.IO.info("Rewriting .beam files") + consolidated_beams = find_consolidated_beams(base_directory) + app_beams = find_app_beams(base_directory) + + Mix.Shell.IO.info(" Found #{length(consolidated_beams)} protocols") + Mix.Shell.IO.info(" Found #{length(app_beams)} app beam files") + + all_beams = Enum.concat(consolidated_beams, app_beams) + total_files = length(all_beams) + chunk_size = ceil(total_files / System.schedulers_online()) + + me = self() + + all_beams + |> Enum.chunk_every(chunk_size) + |> Enum.each(fn chunk -> + Task.async(fn -> + Enum.each(chunk, &apply_and_update_progress(&1, me)) + end) + end) + + block_until_done(0, total_files) + end + + def apply(path) do + erlang_path = String.to_charlist(path) + + with {:ok, forms} <- abstract_code(erlang_path), + rewritten_forms = Abstract.rewrite(forms), + true <- changed?(forms, rewritten_forms), + {:ok, module_name, binary} <- Code.compile(rewritten_forms) do + write_module_beam(path, module_name, binary) + end + end + + defp changed?(same, same), do: false + defp changed?(_, _), do: true + + defp block_until_done(same, same) do + Mix.Shell.IO.info("\n done") + end + + defp block_until_done(current, max) do + receive do + :progress -> :ok + end + + current = current + 1 + IO.write("\r") + percent_complete = format_percent(current, max) + + IO.write(" Applying namespace: #{percent_complete} complete") + block_until_done(current, max) + end + + defp apply_and_update_progress(beam_file, caller) do + apply(beam_file) + send(caller, :progress) + end + + defp find_consolidated_beams(base_directory) do + [base_directory, "**", "consolidated", "*.beam"] + |> Path.join() + |> Path.wildcard() + end + + defp find_app_beams(base_directory) do + namespaced_apps = Enum.join(Namespace.app_names(), ",") + apps_glob = "{#{namespaced_apps}}" + + [base_directory, "lib", apps_glob, "ebin/**", "*.beam"] + |> Path.join() + |> Path.wildcard() + end + + defp write_module_beam(old_path, module_name, binary) do + ebin_path = Path.dirname(old_path) + new_beam_path = Path.join(ebin_path, "#{module_name}.beam") + + with :ok <- File.write(new_beam_path, binary, [:binary, :raw]) do + unless old_path == new_beam_path do + # avoids deleting modules that did not get a new name + # e.g. Elixir.Mix.Task.. etc + File.rm(old_path) + end + end + end + + defp abstract_code(path) do + with {:ok, {_orig_module, code_parts}} <- :beam_lib.chunks(path, [:abstract_code]), + {:ok, {:raw_abstract_v1, forms}} <- Keyword.fetch(code_parts, :abstract_code) do + {:ok, forms} + else + _ -> + {:error, :not_found} + end + end + + defp format_percent(current, max) do + int_val = + (current / max * 100) + |> round() + |> Integer.to_string() + + String.pad_leading("#{int_val}%", 4) + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/boots.ex b/namespace/lib/mix/tasks/namespace/transform/boots.ex new file mode 100644 index 00000000..0fcc0c6e --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/boots.ex @@ -0,0 +1,26 @@ +defmodule Mix.Tasks.Namespace.Transform.Boots do + @moduledoc """ + A transformer that re-builds .boot files by converting a .script file + """ + def apply_to_all(base_directory) do + base_directory + |> find_boot_files() + |> tap(fn boot_files -> + Mix.Shell.IO.info("Rebuilding #{length(boot_files)} boot files") + end) + |> Enum.each(&apply/1) + end + + def apply(file_path) do + file_path + |> Path.rootname() + |> String.to_charlist() + |> :systools.script2boot() + end + + defp find_boot_files(base_directory) do + [base_directory, "releases", "**", "*.script"] + |> Path.join() + |> Path.wildcard() + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/configs.ex b/namespace/lib/mix/tasks/namespace/transform/configs.ex new file mode 100644 index 00000000..2c455a1f --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/configs.ex @@ -0,0 +1,41 @@ +defmodule Mix.Tasks.Namespace.Transform.Configs do + alias Mix.Tasks.Namespace + + def apply_to_all(base_directory) do + base_directory + |> Path.join("**") + |> Path.wildcard() + |> Enum.map(&Path.absname/1) + |> tap(fn paths -> + Mix.Shell.IO.info("Rewriting #{length(paths)} config scripts.") + end) + |> Enum.each(&apply/1) + end + + def apply(path) do + namespaced = + path + |> File.read!() + |> Code.string_to_quoted!() + |> Macro.postwalk(fn + {:__aliases__, meta, alias} -> + namespaced_alias = + alias + |> Module.concat() + |> Namespace.Module.apply() + |> Module.split() + |> Enum.map(&String.to_atom/1) + + {:__aliases__, meta, namespaced_alias} + + atom when is_atom(atom) -> + Namespace.Module.apply(atom) + + ast -> + ast + end) + |> Macro.to_string() + + File.write!(path, namespaced) + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/erlang.ex b/namespace/lib/mix/tasks/namespace/transform/erlang.ex new file mode 100644 index 00000000..77fa59c5 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/erlang.ex @@ -0,0 +1,33 @@ +defmodule Mix.Tasks.Namespace.Transform.Erlang do + @moduledoc """ + Utilities for reading and writing erlang terms from and to text + """ + + def path_to_term(file_path) do + with {:ok, [term]} <- :file.consult(file_path) do + {:ok, term} + end + end + + def path_to_ast(file_path) do + path_charlist = String.to_charlist(file_path) + + with {:ok, [app]} <- :file.consult(path_charlist) do + ast_string = inspect(app) + Code.string_to_quoted(ast_string) + end + end + + def term_to_string(term) do + ~c"~p.~n" + |> :io_lib.format([term]) + |> :lists.flatten() + |> List.to_string() + end + + def ast_to_string(elixir_ast) do + elixir_ast + |> Code.eval_quoted() + |> term_to_string() + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/scripts.ex b/namespace/lib/mix/tasks/namespace/transform/scripts.ex new file mode 100644 index 00000000..37ea5271 --- /dev/null +++ b/namespace/lib/mix/tasks/namespace/transform/scripts.ex @@ -0,0 +1,99 @@ +defmodule Mix.Tasks.Namespace.Transform.Scripts do + @moduledoc """ + A transform that updates any module in .script and .rel files with namespaced versions + """ + + alias Mix.Tasks.Namespace + alias Mix.Tasks.Namespace.Transform + + def apply_to_all(base_directory) do + base_directory + |> find_scripts() + |> tap(fn script_files -> + Mix.Shell.IO.info("Rewriting #{length(script_files)} scripts") + end) + |> Enum.each(&apply/1) + end + + def apply(file_path) do + with {:ok, app_definition} <- Transform.Erlang.path_to_term(file_path), + {:ok, converted} <- convert(app_definition) do + File.write(file_path, converted) + end + end + + @script_names ~w(start.script start_clean.script expert.rel) + defp find_scripts(base_directory) do + scripts_glob = "{" <> Enum.join(@script_names, ",") <> "}" + + [base_directory, "releases", "**", scripts_glob] + |> Path.join() + |> Path.wildcard() + end + + defp convert(app_definition) do + converted = visit(app_definition) + erlang_terms = Transform.Erlang.term_to_string(converted) + + script = """ + %% coding: utf-8 + #{erlang_terms} + """ + + {:ok, script} + end + + # for .rel files + defp visit({:release, release_vsn, erts_vsn, app_versions}) do + fixed_apps = + Enum.map(app_versions, fn {app_name, version, start_type} -> + {Namespace.Module.apply(app_name), version, start_type} + end) + + {:release, release_vsn, erts_vsn, fixed_apps} + end + + defp visit({:script, script_vsn, keys}) do + {:script, script_vsn, Enum.map(keys, &visit/1)} + end + + defp visit({:primLoad, app_list}) do + {:primLoad, Enum.map(app_list, &Namespace.Module.apply/1)} + end + + defp visit({:path, paths}) do + {:path, Enum.map(paths, &Namespace.Path.apply/1)} + end + + defp visit({:apply, {:application, :load, load_apps}}) do + {:apply, {:application, :load, Enum.map(load_apps, &visit/1)}} + end + + defp visit({:apply, {:application, :start_boot, apps_to_start}}) do + {:apply, {:application, :start_boot, Enum.map(apps_to_start, &Namespace.Module.apply/1)}} + end + + defp visit({:application, app_name, app_keys}) do + {:application, Namespace.Module.apply(app_name), Enum.map(app_keys, &visit/1)} + end + + defp visit({:application, app_name}) do + {:application, Namespace.Module.apply(app_name)} + end + + defp visit({:mod, {module_name, args}}) do + {:mod, {Namespace.Module.apply(module_name), Enum.map(args, &visit/1)}} + end + + defp visit({:modules, module_list}) do + {:modules, Enum.map(module_list, &Namespace.Module.apply/1)} + end + + defp visit({:applications, app_names}) do + {:applications, Enum.map(app_names, &Namespace.Module.apply/1)} + end + + defp visit(key_value) do + key_value + end +end diff --git a/namespace/mix.exs b/namespace/mix.exs new file mode 100644 index 00000000..7ce796eb --- /dev/null +++ b/namespace/mix.exs @@ -0,0 +1,28 @@ +defmodule Namespace.MixProject do + use Mix.Project + + def project do + [ + app: :namespace, + version: "0.1.0", + elixir: "~> 1.17", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger, :sasl] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + # {:dep_from_hexpm, "~> 0.3.0"}, + # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"} + ] + end +end diff --git a/namespace/test/namespace_test.exs b/namespace/test/namespace_test.exs new file mode 100644 index 00000000..0a9c1510 --- /dev/null +++ b/namespace/test/namespace_test.exs @@ -0,0 +1,8 @@ +defmodule NamespaceTest do + use ExUnit.Case + doctest Namespace + + test "greets the world" do + assert Namespace.hello() == :world + end +end diff --git a/namespace/test/test_helper.exs b/namespace/test/test_helper.exs new file mode 100644 index 00000000..869559e7 --- /dev/null +++ b/namespace/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() From a279bba1d3fb42039e2ed86d6369de4e81550ac5 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 12 Oct 2024 10:50:39 -0400 Subject: [PATCH 02/18] spike: namespace expert codebase By namespacing the Expert applicaiton as well, we can seamlessly communcate with the Engine application without having have knowledge of the namespaced module names. Doing this naively proved to come with some tradeoffs. The namespacing as it was before would also namespace the application files. This causes the application config to get a little hairy. Lexical will also namespace the config files, but that works because they seem to do this when "packaging", which copies the files into a release like structure. I wanted to be able to run the project as "vanilla" as possible, so to do so, a modification to the namespace mix task allows us to namespace with or without changing the applications. Namespacing the applications is not necessary for the Expert codebase, as the namespacing is done to allow seamless communciation with the engine, and not to avoid collisions with user code. This works nicely, but since we are namespacing the beam files in place, when we make a change and recompile, mix recompiles everything from scratch. To avoid doing this, tooling in the justfile (just is analagous to make) will swap the original compilation artifiacts in and out with the namespaced ones so mix will continue to incrementaly compile. It will namespace everything rather than incrementally namespace, but this happens fast enought that it doesn't seem to be a problem as of now. --- .gitignore | 1 + engine/after.ex | 275 ------------------ engine/before.ex | 272 ----------------- engine/lib/engine/document_symbols.ex | 6 +- engine/mix.exs | 7 +- expert/bin/start | 8 +- expert/lib/expert.ex | 14 +- expert/lib/expert/release.ex | 9 +- expert/lib/expert/runtime.ex | 14 +- expert/mix.exs | 7 + justfile | 43 ++- namespace/lib/mix/tasks/namespace.ex | 22 +- namespace/lib/mix/tasks/namespace/abstract.ex | 12 +- .../lib/mix/tasks/namespace/transform/apps.ex | 56 ++-- .../mix/tasks/namespace/transform/beams.ex | 9 +- 15 files changed, 123 insertions(+), 632 deletions(-) create mode 100644 .gitignore delete mode 100644 engine/after.ex delete mode 100644 engine/before.ex diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e81695ec --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.expert-lsp/ diff --git a/engine/after.ex b/engine/after.ex deleted file mode 100644 index 28738fa8..00000000 --- a/engine/after.ex +++ /dev/null @@ -1,275 +0,0 @@ -[ - {:attribute, 1, :file, {~c"lib/schematic/unification.ex", 1}}, - {:attribute, 1, :module, XPSchematic.Unification}, - {:attribute, 1, :compile, - [:no_auto_import, :debug_info, {:inline, [struct_impl_for: 1]}]}, - {:attribute, 4, :callback, - {{:unify, 3}, - [ - {:type, 4, :fun, - [ - {:type, 4, :product, - [ - {:user_type, 4, :t, []}, - {:type, 4, :term, []}, - {:type, 4, :term, []} - ]}, - {:type, 4, :term, []} - ]} - ]}}, - {:attribute, 5, :callback, - {{:message, 1}, - [ - {:type, 5, :fun, - [{:type, 5, :product, [{:user_type, 5, :t, []}]}, {:type, 5, :term, []}]} - ]}}, - {:attribute, 6, :callback, - {{:kind, 1}, - [ - {:type, 6, :fun, - [{:type, 6, :product, [{:user_type, 6, :t, []}]}, {:type, 6, :term, []}]} - ]}}, - {:attribute, 1, :spec, - {{:impl_for!, 1}, - [ - {:type, 1, :fun, - [{:type, 1, :product, [{:type, 1, :term, []}]}, {:type, 1, :atom, []}]} - ]}}, - {:attribute, 1, :spec, - {{:impl_for, 1}, - [ - {:type, 1, :fun, - [ - {:type, 1, :product, [{:type, 1, :term, []}]}, - {:type, 1, :union, [{:type, 1, :atom, []}, {:atom, 0, nil}]} - ]} - ]}}, - {:attribute, 1, :spec, - {{:__protocol__, 1}, - [ - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :module}]}, - {:atom, 0, XPSchematic.Unification} - ]}, - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :functions}]}, - {:type, 0, :nonempty_list, - [ - {:type, 0, :union, - [ - {:type, 0, :tuple, [{:atom, 0, :unify}, {:integer, 0, 3}]}, - {:type, 0, :tuple, [{:atom, 0, :message}, {:integer, 0, 1}]}, - {:type, 0, :tuple, [{:atom, 0, :kind}, {:integer, 0, 1}]} - ]} - ]} - ]}, - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :consolidated?}]}, - {:type, 1, :boolean, []} - ]}, - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :impls}]}, - {:type, 1, :union, - [ - {:atom, 0, :not_consolidated}, - {:type, 0, :tuple, - [ - {:atom, 0, :consolidated}, - {:type, 0, :list, [{:type, 1, :module, []}]} - ]} - ]} - ]} - ]}}, - {:attribute, 1, :export_type, [t: 0]}, - {:attribute, 1, :type, {:t, {:type, 1, :term, []}, []}}, - {:attribute, 1, :dialyzer, - {:nowarn_function, [__protocol__: 1, impl_for: 1, impl_for!: 1]}}, - {:attribute, 1, :__protocol__, [fallback_to_any: true]}, - {:attribute, 1, :export, - [ - __info__: 1, - __protocol__: 1, - impl_for: 1, - impl_for!: 1, - kind: 1, - message: 1, - unify: 3 - ]}, - {:attribute, 1, :spec, - {{:__info__, 1}, - [ - {:type, 1, :fun, - [ - {:type, 1, :product, - [ - {:type, 1, :union, - [ - {:atom, 1, :attributes}, - {:atom, 1, :compile}, - {:atom, 1, :functions}, - {:atom, 1, :macros}, - {:atom, 1, :md5}, - {:atom, 1, :exports_md5}, - {:atom, 1, :module}, - {:atom, 1, :deprecated}, - {:atom, 1, :struct} - ]} - ]}, - {:type, 1, :any, []} - ]} - ]}}, - {:function, 0, :__info__, 1, - [ - {:clause, 0, [{:atom, 0, :module}], [], - [{:atom, 0, XPSchematic.Unification}]}, - {:clause, 0, [{:atom, 0, :functions}], [], - [ - {:cons, 0, {:tuple, 0, [{:atom, 0, :__protocol__}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for!}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :kind}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :message}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :unify}, {:integer, 0, 3}]}, - {nil, 0}}}}}}} - ]}, - {:clause, 0, [{:atom, 0, :macros}], [], [nil: 0]}, - {:clause, 0, [{:atom, 0, :struct}], [], [{:atom, 0, nil}]}, - {:clause, 0, [{:atom, 0, :exports_md5}], [], - [ - {:bin, 0, - [ - {:bin_element, 0, - {:string, 0, - [89, 58, 13, 228, 237, 62, 86, 234, 224, 87, 49, 205, 30, 99, 0, - 126]}, :default, :default} - ]} - ]}, - {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :attributes}}], [], - [ - {:call, 0, - {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, - [{:atom, 0, XPSchematic.Unification}, {:var, 0, :Key}]} - ]}, - {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :compile}}], [], - [ - {:call, 0, - {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, - [{:atom, 0, XPSchematic.Unification}, {:var, 0, :Key}]} - ]}, - {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :md5}}], [], - [ - {:call, 0, - {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, - [{:atom, 0, XPSchematic.Unification}, {:var, 0, :Key}]} - ]}, - {:clause, 0, [{:atom, 0, :deprecated}], [], [nil: 0]} - ]}, - {:function, 1, :impl_for!, 1, - [ - {:clause, [generated: true, location: 0], [{:var, 1, :_@1}], [], - [{:call, 1, {:atom, 1, :impl_for}, [{:var, 1, :_@1}]}]} - ]}, - {:function, 6, :kind, 1, - [ - {:clause, 6, [{:var, 6, :_@1}], [], - [ - {:call, 6, - {:remote, 6, {:call, 6, {:atom, 6, :impl_for!}, [{:var, 6, :_@1}]}, - {:atom, 6, :kind}}, [{:var, 6, :_@1}]} - ]} - ]}, - {:function, 5, :message, 1, - [ - {:clause, 5, [{:var, 5, :_@1}], [], - [ - {:call, 5, - {:remote, 5, {:call, 5, {:atom, 5, :impl_for!}, [{:var, 5, :_@1}]}, - {:atom, 5, :message}}, [{:var, 5, :_@1}]} - ]} - ]}, - {:function, 4, :unify, 3, - [ - {:clause, 4, [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}], [], - [ - {:call, 4, - {:remote, 4, {:call, 4, {:atom, 4, :impl_for!}, [{:var, 4, :_@1}]}, - {:atom, 4, :unify}}, - [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}]} - ]} - ]}, - {:function, 1, :struct_impl_for, 1, - [ - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], XPSchematic}], [], - [ - {:atom, [generated: true, location: 0], - XPSchematic.Unification.Schematic} - ]}, - {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], - [{:atom, [generated: true, location: 0], XPSchematic.Unification.Any}]} - ]}, - {:function, 1, :impl_for, 1, - [ - {:clause, [generated: true, location: 0], - [ - {:map, 1, - [{:map_field_exact, 1, {:atom, 1, :__struct__}, {:var, 1, :_@1}}]} - ], - [ - [ - {:call, 1, {:remote, 1, {:atom, 1, :erlang}, {:atom, 1, :is_atom}}, - [{:var, 1, :_@1}]} - ] - ], [{:call, 1, {:atom, 1, :struct_impl_for}, [{:var, 1, :_@1}]}]}, - {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], - [{:atom, [generated: true, location: 0], XPSchematic.Unification.Any}]} - ]}, - {:function, 1, :__protocol__, 1, - [ - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :module}], [], - [{:atom, [generated: true, location: 0], XPSchematic.Unification}]}, - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :functions}], [], - [ - {:cons, [generated: true, location: 0], - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :kind}, - {:integer, [generated: true, location: 0], 1} - ]}, - {:cons, [generated: true, location: 0], - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :message}, - {:integer, [generated: true, location: 0], 1} - ]}, - {:cons, [generated: true, location: 0], - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :unify}, - {:integer, [generated: true, location: 0], 3} - ]}, {nil, [generated: true, location: 0]}}}} - ]}, - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :consolidated?}], [], - [{:atom, [generated: true, location: 0], true}]}, - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :impls}], [], - [ - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :consolidated}, - {:cons, [generated: true, location: 0], - {:atom, [generated: true, location: 0], Any}, - {:cons, [generated: true, location: 0], - {:atom, [generated: true, location: 0], XPSchematic}, - {nil, [generated: true, location: 0]}}} - ]} - ]} - ]} -] \ No newline at end of file diff --git a/engine/before.ex b/engine/before.ex deleted file mode 100644 index 5d2122b1..00000000 --- a/engine/before.ex +++ /dev/null @@ -1,272 +0,0 @@ -[ - {:attribute, 1, :file, {~c"lib/schematic/unification.ex", 1}}, - {:attribute, 1, :module, Schematic.Unification}, - {:attribute, 1, :compile, - [:no_auto_import, :debug_info, {:inline, [struct_impl_for: 1]}]}, - {:attribute, 4, :callback, - {{:unify, 3}, - [ - {:type, 4, :fun, - [ - {:type, 4, :product, - [ - {:user_type, 4, :t, []}, - {:type, 4, :term, []}, - {:type, 4, :term, []} - ]}, - {:type, 4, :term, []} - ]} - ]}}, - {:attribute, 5, :callback, - {{:message, 1}, - [ - {:type, 5, :fun, - [{:type, 5, :product, [{:user_type, 5, :t, []}]}, {:type, 5, :term, []}]} - ]}}, - {:attribute, 6, :callback, - {{:kind, 1}, - [ - {:type, 6, :fun, - [{:type, 6, :product, [{:user_type, 6, :t, []}]}, {:type, 6, :term, []}]} - ]}}, - {:attribute, 1, :spec, - {{:impl_for!, 1}, - [ - {:type, 1, :fun, - [{:type, 1, :product, [{:type, 1, :term, []}]}, {:type, 1, :atom, []}]} - ]}}, - {:attribute, 1, :spec, - {{:impl_for, 1}, - [ - {:type, 1, :fun, - [ - {:type, 1, :product, [{:type, 1, :term, []}]}, - {:type, 1, :union, [{:type, 1, :atom, []}, {:atom, 0, nil}]} - ]} - ]}}, - {:attribute, 1, :spec, - {{:__protocol__, 1}, - [ - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :module}]}, - {:atom, 0, Schematic.Unification} - ]}, - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :functions}]}, - {:type, 0, :nonempty_list, - [ - {:type, 0, :union, - [ - {:type, 0, :tuple, [{:atom, 0, :unify}, {:integer, 0, 3}]}, - {:type, 0, :tuple, [{:atom, 0, :message}, {:integer, 0, 1}]}, - {:type, 0, :tuple, [{:atom, 0, :kind}, {:integer, 0, 1}]} - ]} - ]} - ]}, - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :consolidated?}]}, - {:type, 1, :boolean, []} - ]}, - {:type, 1, :fun, - [ - {:type, 1, :product, [{:atom, 0, :impls}]}, - {:type, 1, :union, - [ - {:atom, 0, :not_consolidated}, - {:type, 0, :tuple, - [ - {:atom, 0, :consolidated}, - {:type, 0, :list, [{:type, 1, :module, []}]} - ]} - ]} - ]} - ]}}, - {:attribute, 1, :export_type, [t: 0]}, - {:attribute, 1, :type, {:t, {:type, 1, :term, []}, []}}, - {:attribute, 1, :dialyzer, - {:nowarn_function, [__protocol__: 1, impl_for: 1, impl_for!: 1]}}, - {:attribute, 1, :__protocol__, [fallback_to_any: true]}, - {:attribute, 1, :export, - [ - __info__: 1, - __protocol__: 1, - impl_for: 1, - impl_for!: 1, - kind: 1, - message: 1, - unify: 3 - ]}, - {:attribute, 1, :spec, - {{:__info__, 1}, - [ - {:type, 1, :fun, - [ - {:type, 1, :product, - [ - {:type, 1, :union, - [ - {:atom, 1, :attributes}, - {:atom, 1, :compile}, - {:atom, 1, :functions}, - {:atom, 1, :macros}, - {:atom, 1, :md5}, - {:atom, 1, :exports_md5}, - {:atom, 1, :module}, - {:atom, 1, :deprecated}, - {:atom, 1, :struct} - ]} - ]}, - {:type, 1, :any, []} - ]} - ]}}, - {:function, 0, :__info__, 1, - [ - {:clause, 0, [{:atom, 0, :module}], [], - [{:atom, 0, Schematic.Unification}]}, - {:clause, 0, [{:atom, 0, :functions}], [], - [ - {:cons, 0, {:tuple, 0, [{:atom, 0, :__protocol__}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :impl_for!}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :kind}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :message}, {:integer, 0, 1}]}, - {:cons, 0, {:tuple, 0, [{:atom, 0, :unify}, {:integer, 0, 3}]}, - {nil, 0}}}}}}} - ]}, - {:clause, 0, [{:atom, 0, :macros}], [], [nil: 0]}, - {:clause, 0, [{:atom, 0, :struct}], [], [{:atom, 0, nil}]}, - {:clause, 0, [{:atom, 0, :exports_md5}], [], - [ - {:bin, 0, - [ - {:bin_element, 0, - {:string, 0, - [89, 58, 13, 228, 237, 62, 86, 234, 224, 87, 49, 205, 30, 99, 0, - 126]}, :default, :default} - ]} - ]}, - {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :attributes}}], [], - [ - {:call, 0, - {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, - [{:atom, 0, Schematic.Unification}, {:var, 0, :Key}]} - ]}, - {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :compile}}], [], - [ - {:call, 0, - {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, - [{:atom, 0, Schematic.Unification}, {:var, 0, :Key}]} - ]}, - {:clause, 0, [{:match, 0, {:var, 0, :Key}, {:atom, 0, :md5}}], [], - [ - {:call, 0, - {:remote, 0, {:atom, 0, :erlang}, {:atom, 0, :get_module_info}}, - [{:atom, 0, Schematic.Unification}, {:var, 0, :Key}]} - ]}, - {:clause, 0, [{:atom, 0, :deprecated}], [], [nil: 0]} - ]}, - {:function, 1, :impl_for!, 1, - [ - {:clause, [generated: true, location: 0], [{:var, 1, :_@1}], [], - [{:call, 1, {:atom, 1, :impl_for}, [{:var, 1, :_@1}]}]} - ]}, - {:function, 6, :kind, 1, - [ - {:clause, 6, [{:var, 6, :_@1}], [], - [ - {:call, 6, - {:remote, 6, {:call, 6, {:atom, 6, :impl_for!}, [{:var, 6, :_@1}]}, - {:atom, 6, :kind}}, [{:var, 6, :_@1}]} - ]} - ]}, - {:function, 5, :message, 1, - [ - {:clause, 5, [{:var, 5, :_@1}], [], - [ - {:call, 5, - {:remote, 5, {:call, 5, {:atom, 5, :impl_for!}, [{:var, 5, :_@1}]}, - {:atom, 5, :message}}, [{:var, 5, :_@1}]} - ]} - ]}, - {:function, 4, :unify, 3, - [ - {:clause, 4, [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}], [], - [ - {:call, 4, - {:remote, 4, {:call, 4, {:atom, 4, :impl_for!}, [{:var, 4, :_@1}]}, - {:atom, 4, :unify}}, - [{:var, 4, :_@1}, {:var, 4, :_@2}, {:var, 4, :_@3}]} - ]} - ]}, - {:function, 1, :struct_impl_for, 1, - [ - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], Schematic}], [], - [{:atom, [generated: true, location: 0], Schematic.Unification.Schematic}]}, - {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], - [{:atom, [generated: true, location: 0], Schematic.Unification.Any}]} - ]}, - {:function, 1, :impl_for, 1, - [ - {:clause, [generated: true, location: 0], - [ - {:map, 1, - [{:map_field_exact, 1, {:atom, 1, :__struct__}, {:var, 1, :_@1}}]} - ], - [ - [ - {:call, 1, {:remote, 1, {:atom, 1, :erlang}, {:atom, 1, :is_atom}}, - [{:var, 1, :_@1}]} - ] - ], [{:call, 1, {:atom, 1, :struct_impl_for}, [{:var, 1, :_@1}]}]}, - {:clause, [generated: true, location: 0], [{:var, 0, :_}], [], - [{:atom, [generated: true, location: 0], Schematic.Unification.Any}]} - ]}, - {:function, 1, :__protocol__, 1, - [ - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :module}], [], - [{:atom, [generated: true, location: 0], Schematic.Unification}]}, - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :functions}], [], - [ - {:cons, [generated: true, location: 0], - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :kind}, - {:integer, [generated: true, location: 0], 1} - ]}, - {:cons, [generated: true, location: 0], - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :message}, - {:integer, [generated: true, location: 0], 1} - ]}, - {:cons, [generated: true, location: 0], - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :unify}, - {:integer, [generated: true, location: 0], 3} - ]}, {nil, [generated: true, location: 0]}}}} - ]}, - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :consolidated?}], [], - [{:atom, [generated: true, location: 0], true}]}, - {:clause, [generated: true, location: 0], - [{:atom, [generated: true, location: 0], :impls}], [], - [ - {:tuple, [generated: true, location: 0], - [ - {:atom, [generated: true, location: 0], :consolidated}, - {:cons, [generated: true, location: 0], - {:atom, [generated: true, location: 0], Any}, - {:cons, [generated: true, location: 0], - {:atom, [generated: true, location: 0], Schematic}, - {nil, [generated: true, location: 0]}}} - ]} - ]} - ]} -] \ No newline at end of file diff --git a/engine/lib/engine/document_symbols.ex b/engine/lib/engine/document_symbols.ex index 1eacfcfa..47305fad 100644 --- a/engine/lib/engine/document_symbols.ex +++ b/engine/lib/engine/document_symbols.ex @@ -34,11 +34,7 @@ defmodule Engine.DocumentSymbol do ast end - for %DocumentSymbol{} = ds <- List.wrap(walker(ast, nil)) do - {:ok, dumped} = Schematic.dump(DocumentSymbol.schema(), ds) - - dumped - end + List.wrap(walker(ast, nil)) end defp walker([{{:__literal__, _, [:do]}, {_, _, _exprs} = ast}], mod) do diff --git a/engine/mix.exs b/engine/mix.exs index 256f5b4c..1b18b36a 100644 --- a/engine/mix.exs +++ b/engine/mix.exs @@ -8,7 +8,12 @@ defmodule Engine.MixProject do elixir: "~> 1.17", start_permanent: Mix.env() == :prod, aliases: [ - build: ["cmd rm -rf _build/#{Mix.env()}", "compile", "namespace _build/#{Mix.env()}"] + namespace: "namespace --apps", + build: [ + "cmd rm -rf _build/#{Mix.env()}", + "compile", + "namespace --apps --directory _build/#{Mix.env()}" + ] ], deps: deps() ] diff --git a/expert/bin/start b/expert/bin/start index c946f14e..871d4cf0 100755 --- a/expert/bin/start +++ b/expert/bin/start @@ -2,10 +2,4 @@ cd "$(dirname "$0")"/.. || exit 1 -( - cd ../engine || exit - - mix build -) - -EXPERT_ENGINE_PATH="../engine/_build/${MIX_ENV:-dev}/" mix run --no-halt -e "Application.ensure_all_started(:expert)" -- "$@" +EXPERT_ENGINE_PATH="../engine/_build/${MIX_ENV:-dev}/" mix run --no-compile --no-halt -e "Application.ensure_all_started(:expert)" -- "$@" diff --git a/expert/lib/expert.ex b/expert/lib/expert.ex index 81daf6fa..5d9ee741 100644 --- a/expert/lib/expert.ex +++ b/expert/lib/expert.ex @@ -110,16 +110,8 @@ defmodule Expert do symbols = Expert.Runtime.execute! lsp.assigns.runtime do - # we have to call the namespaced module name - XPert.DocumentSymbol.fetch(doc) + Engine.DocumentSymbol.fetch(doc) end - |> Enum.map(fn ds -> - # we also have to serialize and deserialize to send structs between nodes that - # might be namespaced - - {:ok, unified} = Schematic.unify(GenLSP.Structures.DocumentSymbol.schema(), ds) - unified - end) # which then will get serialized again on the way out # we could potentially namespace our app too, but i think that @@ -145,13 +137,13 @@ defmodule Expert do {:noreply, lsp} end - def handle_info({:runtime_ready, name, runtime_pid}, lsp) do + def handle_info({:runtime_ready, _name, runtime_pid}, lsp) do Runtime.compile(runtime_pid) {:noreply, assign(lsp, ready: true, runtime: runtime_pid)} end - def handle_info({:compiler_result, name, result}, lsp) do + def handle_info({:compiler_result, _name, result}, lsp) do case result do {status, diagnostics} when status in [:ok, :noop] -> per_file = diff --git a/expert/lib/expert/release.ex b/expert/lib/expert/release.ex index 2588279f..5d1dd046 100644 --- a/expert/lib/expert/release.ex +++ b/expert/lib/expert/release.ex @@ -10,7 +10,14 @@ defmodule Expert.Release do ] ) - source = Path.join([engine_path, "_build/prod"]) + {_, 0} = + System.cmd("mix", ["namespace"], + env: [ + {"MIX_ENV", to_string(Mix.env())} + ] + ) + + source = Path.join([engine_path, "_build/#{Mix.env()}"]) dest = Path.join([ diff --git a/expert/lib/expert/runtime.ex b/expert/lib/expert/runtime.ex index 674970fb..2043c10e 100644 --- a/expert/lib/expert/runtime.ex +++ b/expert/lib/expert/runtime.ex @@ -2,7 +2,6 @@ defmodule Expert.Runtime do @moduledoc false use GenServer - @env Mix.env() defguardp is_ready(state) when is_map_key(state, :node) def start_link(opts) do @@ -130,9 +129,6 @@ defmodule Expert.Runtime do |> Path.join("cmd") |> Path.absname() - engine_path = - System.get_env("EXPERT_ENGINE_PATH", to_string(dir)) |> Path.expand() - env = [ {~c"LSP", ~c"expert"}, @@ -145,6 +141,9 @@ defmodule Expert.Runtime do {~c"PATH", String.to_charlist(new_path)} ] + engine_path = + System.get_env("EXPERT_ENGINE_PATH", to_string(dir)) |> Path.expand() + consolidated = Path.wildcard(Path.join(engine_path, "lib/*/{consolidated}")) |> Enum.flat_map(fn ep -> ["-pa", ep] end) @@ -157,11 +156,6 @@ defmodule Expert.Runtime do args = [elixir_exe] ++ - if @env == :test do - ["--erl", "-kernel prevent_overlapping_partitions false"] - else - [] - end ++ engine_path_args ++ [ "--no-halt", @@ -250,7 +244,7 @@ defmodule Expert.Runtime do |> Keyword.put(:from, self()) with {:badrpc, _error} <- - :rpc.call(node, XPert.Worker, :enqueue_compiler, [opts]) do + :rpc.call(node, Engine.Worker, :enqueue_compiler, [opts]) do :error end diff --git a/expert/mix.exs b/expert/mix.exs index c57bb7c8..b89b4226 100644 --- a/expert/mix.exs +++ b/expert/mix.exs @@ -9,6 +9,7 @@ defmodule Expert.MixProject do start_permanent: Mix.env() == :prod, elixirc_paths: elixirc_paths(Mix.env()), releases: releases(), + aliases: aliases(), default_release: :expert, deps: deps() ] @@ -43,6 +44,12 @@ defmodule Expert.MixProject do defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] + defp aliases() do + [ + namespace: "namespace" + ] + end + # Run "mix help deps" to learn about dependencies. defp deps do [ diff --git a/justfile b/justfile index b6e6d95d..5e254342 100644 --- a/justfile +++ b/justfile @@ -3,20 +3,39 @@ deps project: cd {{project}} mix deps.get -compile project: +build project: #!/usr/bin/env bash + set -euo pipefail + cd {{project}} + mix_env="${MIX_ENV:-dev}" + build_dir="_build/$mix_env" + safe_dir="_build/$mix_env-safe" + # create our safekeeping area + mkdir -p "$safe_dir" + # delete what is currently in the build dir + rm -rf "$build_dir" + # move our build artifacts from safekeeping to the build area + cp -a "$safe_dir/." "$build_dir/" + # compile the safe kept code, respects incremental compilation mix compile + # prep the safe area for new code + rm -rf "$safe_dir/" + # copy new code in the safe area + cp -a "$build_dir/." "$safe_dir/" + # namespace the new code + mix namespace --directory "$build_dir" -build: - #!/usr/bin/env bash - cd engine - mix build - -start: +start *opts="--port 9000": (build "engine") (build "expert") #!/usr/bin/env bash cd expert - bin/start --port 9000 + + # no compile is important so it doesn't mess up the namespacing + EXPERT_ENGINE_PATH="../engine/_build/${MIX_ENV:-dev}/" mix run \ + --no-compile \ + --no-halt \ + -e "Application.ensure_all_started(:expert)" \ + -- {{opts}} test project: #!/usr/bin/env bash @@ -29,7 +48,7 @@ format project: mix format [unix] -build-local: +release-local: #!/usr/bin/env bash cd expert case "{{os()}}-{{arch()}}" in @@ -49,14 +68,14 @@ build-local: EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="$target" MIX_ENV=prod mix release [windows] -build-local: +release-local: # idk actually how to set env vars like this on windows, might crash EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release -build-all: +release-all: cd expert EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release -build-plain: +release-plain: cd expert MIX_ENV=prod mix release plain diff --git a/namespace/lib/mix/tasks/namespace.ex b/namespace/lib/mix/tasks/namespace.ex index 27933e9e..5a1e2a06 100644 --- a/namespace/lib/mix/tasks/namespace.ex +++ b/namespace/lib/mix/tasks/namespace.ex @@ -19,7 +19,8 @@ defmodule Mix.Tasks.Namespace do # by this task. Plugin discovery uses this task, which happens after # namespacing. @extra_apps %{ - "engine" => "Engine" + "engine" => "Engine", + "expert" => "Expert" } defp deps_apps() do @@ -30,18 +31,29 @@ defmodule Mix.Tasks.Namespace do require Logger - def run([base_directory]) do + def run(argv) do + {options, _rest} = + OptionParser.parse!(argv, + strict: [directory: :string, apps: :boolean] + ) + + base_directory = options[:directory] Process.put(:deps_apps, deps_apps()) - Transform.Apps.apply_to_all(base_directory) + + Transform.Apps.apply_to_all(base_directory, options[:apps]) + Transform.Beams.apply_to_all(base_directory) + # Transform.Configs.apply_to_all(base_directory) # consolidated - # Transform.Beams.apply(base_directory) Transform.Scripts.apply_to_all(base_directory) # The boot file transform just turns script files into boot files # so it must come after the script file transform Transform.Boots.apply_to_all(base_directory) - Transform.AppDirectories.apply_to_all(base_directory) + + if options[:apps] do + Transform.AppDirectories.apply_to_all(base_directory) + end end def app_names() do diff --git a/namespace/lib/mix/tasks/namespace/abstract.ex b/namespace/lib/mix/tasks/namespace/abstract.ex index 0f7d2d67..7b0cf33f 100644 --- a/namespace/lib/mix/tasks/namespace/abstract.ex +++ b/namespace/lib/mix/tasks/namespace/abstract.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Namespace.Abstract do {:attribute, anno, :__impl__, rewrite(attrs)} end - defp do_rewrite({:function, anno, name, arity, clauses} = full) do + defp do_rewrite({:function, anno, name, arity, clauses}) do {:function, anno, name, arity, rewrite(clauses)} end @@ -90,7 +90,7 @@ defmodule Mix.Tasks.Namespace.Abstract do # 8.2 Atomic Literals # only rewrite atoms, since they might be modules - defp do_rewrite({:atom, anno, literal}) do + defp do_rewrite({:atom, anno, literal}) when literal != :expert do {:atom, anno, rewrite_module(literal)} end @@ -151,11 +151,11 @@ defmodule Mix.Tasks.Namespace.Abstract do {:catch, anno, rewrite(expression)} end - defp do_rewrite({:fun, anno, {:function, name, arity}} = full) do + defp do_rewrite({:fun, anno, {:function, name, arity}}) do {:fun, anno, {:function, rewrite(name), arity}} end - defp do_rewrite({:fun, anno, {:function, module, name, arity}} = full) do + defp do_rewrite({:fun, anno, {:function, module, name, arity}}) do {:fun, anno, {:function, rewrite(module), rewrite(name), arity}} end @@ -163,11 +163,11 @@ defmodule Mix.Tasks.Namespace.Abstract do {:fun, anno, {:clauses, rewrite(clauses)}} end - defp do_rewrite({:named_fun, anno, name, clauses} = full) do + defp do_rewrite({:named_fun, anno, name, clauses}) do {:named_fun, anno, rewrite(name), rewrite(clauses)} end - defp do_rewrite({:call, anno, {:remote, remote_anno, module, fn_name}, args} = full) do + defp do_rewrite({:call, anno, {:remote, remote_anno, module, fn_name}, args}) do {:call, anno, {:remote, remote_anno, rewrite(module), fn_name}, rewrite(args)} end diff --git a/namespace/lib/mix/tasks/namespace/transform/apps.ex b/namespace/lib/mix/tasks/namespace/transform/apps.ex index a3c1e938..7fdf233f 100644 --- a/namespace/lib/mix/tasks/namespace/transform/apps.ex +++ b/namespace/lib/mix/tasks/namespace/transform/apps.ex @@ -4,19 +4,20 @@ defmodule Mix.Tasks.Namespace.Transform.Apps do """ alias Mix.Tasks.Namespace alias Mix.Tasks.Namespace.Transform + import Kernel, except: [apply: 2] - def apply_to_all(base_directory) do + def apply_to_all(base_directory, namespace_app) do base_directory |> find_app_files() |> tap(fn app_files -> Mix.Shell.IO.info("Rewriting #{length(app_files)} app files") end) - |> Enum.each(&apply/1) + |> Enum.each(fn f -> apply(f, namespace_app) end) end - def apply(file_path) do + def apply(file_path, namespace_app) do with {:ok, app_definition} <- Transform.Erlang.path_to_term(file_path), - {:ok, converted} <- convert(app_definition), + {:ok, converted} <- convert(app_definition, namespace_app), :ok <- File.write(file_path, converted) do app_name = file_path @@ -24,15 +25,17 @@ defmodule Mix.Tasks.Namespace.Transform.Apps do |> Path.rootname() |> String.to_atom() - namespaced_app_name = Namespace.Module.apply(app_name) - new_filename = "#{namespaced_app_name}.app" + if namespace_app do + namespaced_app_name = Namespace.Module.apply(app_name) + new_filename = "#{namespaced_app_name}.app" - new_file_path = - file_path - |> Path.dirname() - |> Path.join(new_filename) + new_file_path = + file_path + |> Path.dirname() + |> Path.join(new_filename) - File.rename!(file_path, new_file_path) + File.rename!(file_path, new_file_path) + end end end @@ -44,36 +47,47 @@ defmodule Mix.Tasks.Namespace.Transform.Apps do |> Path.wildcard() end - defp convert(app_definition) do + defp convert(app_definition, namespace_app) do erlang_terms = app_definition - |> visit() + |> visit(namespace_app) |> Transform.Erlang.term_to_string() {:ok, erlang_terms} end - defp visit({:application, app_name, keys}) do - {:application, Namespace.Module.apply(app_name), Enum.map(keys, &visit/1)} + defp visit({:application, app_name, keys}, namespace_app) do + app = + if namespace_app do + Namespace.Module.apply(app_name) + else + app_name + end + + {:application, app, Enum.map(keys, fn k -> visit(k, namespace_app) end)} end - defp visit({:applications, app_list}) do - {:applications, Enum.map(app_list, &Namespace.Module.apply/1)} + defp visit({:applications, app_list} = original, namespace_app) do + if namespace_app do + {:applications, Enum.map(app_list, &Namespace.Module.apply/1)} + else + original + end end - defp visit({:modules, module_list}) do + defp visit({:modules, module_list}, _) do {:modules, Enum.map(module_list, &Namespace.Module.apply/1)} end - defp visit({:description, desc}) do + defp visit({:description, desc}, _) do {:description, desc ++ ~c" namespaced by expert."} end - defp visit({:mod, {module_name, args}}) do + defp visit({:mod, {module_name, args}}, _) do {:mod, {Namespace.Module.apply(module_name), args}} end - defp visit(key_value) do + defp visit(key_value, _) do key_value end end diff --git a/namespace/lib/mix/tasks/namespace/transform/beams.ex b/namespace/lib/mix/tasks/namespace/transform/beams.ex index c35d9047..7f921d01 100644 --- a/namespace/lib/mix/tasks/namespace/transform/beams.ex +++ b/namespace/lib/mix/tasks/namespace/transform/beams.ex @@ -17,17 +17,14 @@ defmodule Mix.Tasks.Namespace.Transform.Beams do all_beams = Enum.concat(consolidated_beams, app_beams) total_files = length(all_beams) - chunk_size = ceil(total_files / System.schedulers_online()) me = self() all_beams - |> Enum.chunk_every(chunk_size) - |> Enum.each(fn chunk -> - Task.async(fn -> - Enum.each(chunk, &apply_and_update_progress(&1, me)) - end) + |> Task.async_stream(fn beam -> + apply_and_update_progress(beam, me) end) + |> Stream.run() block_until_done(0, total_files) end From 866b9d4b2a254376c167d6cf2054332d585b888c Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Fri, 25 Oct 2024 16:12:24 -0400 Subject: [PATCH 03/18] spike(namespace): parameterize transformers This patch primarily parameterizes the transformers to accept the apps and the root modules they need to work on as parameters instead as global state. This increases the flexibility of the project by hoisting this configurability to the boundary, allowing it to be overridden/customized by CLI options. This was accomplished by passing an "opts" parameter to each transformer function (which were consequently renamed from "apply" to "run" to avoid a naming collision with Kernel.apply/2). Incidental changes are moving the transformers from the mix task directory to the regular lib directory and to add unit tests to ensure that there weren't any regressions. Something to note, I don't think that the `configs`, `scripts`, or `boots` transformers are needed any longer, but that conclusion should be double checked. --- namespace/lib/mix/tasks/namespace.ex | 95 +++-------- namespace/lib/mix/tasks/namespace/path.ex | 29 ---- .../namespace/transform/app_directories.ex | 25 --- .../lib/mix/tasks/namespace/transform/apps.ex | 93 ----------- .../mix/tasks/namespace/transform/configs.ex | 41 ----- .../mix/tasks/namespace/transform/scripts.ex | 99 ------------ .../lib/{mix/tasks => }/namespace/abstract.ex | 149 ++++++++++-------- .../lib/{mix/tasks => }/namespace/code.ex | 2 +- .../transform => namespace}/erlang.ex | 2 +- .../lib/{mix/tasks => }/namespace/module.ex | 23 ++- namespace/lib/namespace/path.ex | 27 ++++ .../namespace/transform/app_directories.ex | 18 +++ namespace/lib/namespace/transform/apps.ex | 85 ++++++++++ .../tasks => }/namespace/transform/beams.ex | 34 ++-- .../tasks => }/namespace/transform/boots.ex | 8 +- namespace/lib/namespace/transform/configs.ex | 41 +++++ namespace/lib/namespace/transform/scripts.ex | 96 +++++++++++ namespace/mix.exs | 9 ++ namespace/mix.lock | 2 + namespace/test/namespace/abstract_test.exs | 57 +++++++ namespace/test/namespace/module_test.exs | 24 +++ namespace/test/namespace/path_test.exs | 14 ++ .../transform/app_directories_test.exs | 26 +++ .../test/namespace/transform/apps_test.exs | 62 ++++++++ .../test/namespace/transform/beams_test.exs | 96 +++++++++++ .../test/namespace/transform/boots_test.exs | 6 + namespace/test/namespace_test.exs | 8 - namespace/test/support/fixtures/forms.ex | 35 ++++ namespace/test/test_helper.exs | 2 +- 29 files changed, 736 insertions(+), 472 deletions(-) delete mode 100644 namespace/lib/mix/tasks/namespace/path.ex delete mode 100644 namespace/lib/mix/tasks/namespace/transform/app_directories.ex delete mode 100644 namespace/lib/mix/tasks/namespace/transform/apps.ex delete mode 100644 namespace/lib/mix/tasks/namespace/transform/configs.ex delete mode 100644 namespace/lib/mix/tasks/namespace/transform/scripts.ex rename namespace/lib/{mix/tasks => }/namespace/abstract.ex (53%) rename namespace/lib/{mix/tasks => }/namespace/code.ex (70%) rename namespace/lib/{mix/tasks/namespace/transform => namespace}/erlang.ex (93%) rename namespace/lib/{mix/tasks => }/namespace/module.ex (80%) create mode 100644 namespace/lib/namespace/path.ex create mode 100644 namespace/lib/namespace/transform/app_directories.ex create mode 100644 namespace/lib/namespace/transform/apps.ex rename namespace/lib/{mix/tasks => }/namespace/transform/beams.ex (80%) rename namespace/lib/{mix/tasks => }/namespace/transform/boots.ex (76%) create mode 100644 namespace/lib/namespace/transform/configs.ex create mode 100644 namespace/lib/namespace/transform/scripts.ex create mode 100644 namespace/mix.lock create mode 100644 namespace/test/namespace/abstract_test.exs create mode 100644 namespace/test/namespace/module_test.exs create mode 100644 namespace/test/namespace/path_test.exs create mode 100644 namespace/test/namespace/transform/app_directories_test.exs create mode 100644 namespace/test/namespace/transform/apps_test.exs create mode 100644 namespace/test/namespace/transform/beams_test.exs create mode 100644 namespace/test/namespace/transform/boots_test.exs delete mode 100644 namespace/test/namespace_test.exs create mode 100644 namespace/test/support/fixtures/forms.ex diff --git a/namespace/lib/mix/tasks/namespace.ex b/namespace/lib/mix/tasks/namespace.ex index 5a1e2a06..be55e7ad 100644 --- a/namespace/lib/mix/tasks/namespace.ex +++ b/namespace/lib/mix/tasks/namespace.ex @@ -10,75 +10,49 @@ defmodule Mix.Tasks.Namespace do This task takes a single argument, which is the full path to the release. """ - alias Mix.Tasks.Namespace.Transform use Mix.Task - @dev_deps [:namespace] - - # These app names and root modules are strings to avoid them being namespaced - # by this task. Plugin discovery uses this task, which happens after - # namespacing. - @extra_apps %{ - "engine" => "Engine", - "expert" => "Expert" - } - - defp deps_apps() do - Mix.Project.deps_apps() - |> Kernel.--(@dev_deps) - |> Enum.map(&to_string/1) - end - require Logger def run(argv) do {options, _rest} = OptionParser.parse!(argv, - strict: [directory: :string, apps: :boolean] + strict: [ + directory: :string, + apps: :boolean, + include_app: :keep, + include_root: :keep, + exclude_app: :keep, + exclude_root: :keep + ] ) base_directory = options[:directory] - Process.put(:deps_apps, deps_apps()) - Transform.Apps.apply_to_all(base_directory, options[:apps]) + include_apps = Keyword.get_values(options, :include_app) |> Enum.map(&String.to_atom/1) + include_roots = Keyword.get_values(options, :include_root) |> Enum.map(&Module.concat([&1])) + exclude_apps = Keyword.get_values(options, :exclude_app) |> Enum.map(&String.to_atom/1) + exclude_roots = Keyword.get_values(options, :exclude_root) |> Enum.map(&Module.concat([&1])) - Transform.Beams.apply_to_all(base_directory) - # Transform.Configs.apply_to_all(base_directory) - # consolidated - Transform.Scripts.apply_to_all(base_directory) - # The boot file transform just turns script files into boot files - # so it must come after the script file transform - Transform.Boots.apply_to_all(base_directory) + apps = (Mix.Project.deps_apps() ++ include_apps) -- exclude_apps - if options[:apps] do - Transform.AppDirectories.apply_to_all(base_directory) - end - end + roots_from_apps = apps |> root_modules_for_apps() |> Map.values() |> List.flatten() |> Enum.uniq() - def app_names() do - Map.keys(app_to_root_modules()) - end + roots = (roots_from_apps ++ include_roots) -- exclude_roots - def root_modules() do - app_to_root_modules() - |> Map.values() - |> List.flatten() - end - def app_to_root_modules() do - case :persistent_term.get(__MODULE__, :not_loaded) do - :not_loaded -> - init() + opts = [apps: apps, roots: roots, do_apps: options[:apps]] - term -> - term - end - end + Namespace.Transform.Apps.run_all(base_directory, options[:apps], opts) + + Namespace.Transform.Beams.run_all(base_directory, options[:apps], opts) + # Transform.Configs.run_all(base_directory) + # consolidated - defp register_mappings(app_to_root_modules) do - :persistent_term.put(__MODULE__, app_to_root_modules) - app_to_root_modules + if options[:apps] do + Namespace.Transform.AppDirectories.run_all(base_directory, opts) + end end defp root_modules_for_apps(deps_apps) do @@ -87,7 +61,7 @@ defmodule Mix.Tasks.Namespace do all_modules = app_modules(app_name) case Enum.filter(all_modules, fn module -> length(safe_split_module(module)) == 1 end) do - [] -> {app_name, [Expert]} + [] -> {app_name, []} root_modules -> {app_name, root_modules} end end) @@ -114,25 +88,6 @@ defmodule Mix.Tasks.Namespace do end end - defp extra_apps do - Map.new(@extra_apps, fn {k, v} -> - root_module = - v - |> List.wrap() - |> Module.concat() - - {String.to_atom(k), [root_module]} - end) - end - - defp init() do - Process.get(:deps_apps) - |> Enum.map(&String.to_atom/1) - |> root_modules_for_apps() - |> Map.merge(extra_apps()) - |> register_mappings() - end - def safe_split(module, opts \\ []) def safe_split(module, opts) when is_atom(module) do diff --git a/namespace/lib/mix/tasks/namespace/path.ex b/namespace/lib/mix/tasks/namespace/path.ex deleted file mode 100644 index aaf1778f..00000000 --- a/namespace/lib/mix/tasks/namespace/path.ex +++ /dev/null @@ -1,29 +0,0 @@ -defmodule Mix.Tasks.Namespace.Path do - alias Mix.Tasks.Namespace - - def apply(path) when is_list(path) do - path - |> List.to_string() - |> apply() - |> String.to_charlist() - end - - def apply(path) when is_binary(path) do - path - |> Path.split() - |> Enum.map(&replace_namespaced_apps/1) - |> Path.join() - end - - defp replace_namespaced_apps(path_component) do - Enum.reduce(Namespace.app_names(), path_component, fn app_name, path -> - if path == Atom.to_string(app_name) do - app_name - |> Namespace.Module.apply() - |> Atom.to_string() - else - path - end - end) - end -end diff --git a/namespace/lib/mix/tasks/namespace/transform/app_directories.ex b/namespace/lib/mix/tasks/namespace/transform/app_directories.ex deleted file mode 100644 index c8f844d3..00000000 --- a/namespace/lib/mix/tasks/namespace/transform/app_directories.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Mix.Tasks.Namespace.Transform.AppDirectories do - alias Mix.Tasks.Namespace - - def apply_to_all(base_directory) do - base_directory - |> find_app_directories() - |> Enum.each(&apply/1) - end - - def apply(app_path) do - namespaced_app_path = Namespace.Path.apply(app_path) - - with {:ok, _} <- File.rm_rf(namespaced_app_path) do - File.rename!(app_path, namespaced_app_path) - end - end - - defp find_app_directories(base_directory) do - app_globs = Enum.join(Namespace.app_names(), "*,") - - [base_directory, "lib", "{" <> app_globs <> "*}"] - |> Path.join() - |> Path.wildcard() - end -end diff --git a/namespace/lib/mix/tasks/namespace/transform/apps.ex b/namespace/lib/mix/tasks/namespace/transform/apps.ex deleted file mode 100644 index 7fdf233f..00000000 --- a/namespace/lib/mix/tasks/namespace/transform/apps.ex +++ /dev/null @@ -1,93 +0,0 @@ -defmodule Mix.Tasks.Namespace.Transform.Apps do - @moduledoc """ - Applies namespacing to all modules defined in .app files - """ - alias Mix.Tasks.Namespace - alias Mix.Tasks.Namespace.Transform - import Kernel, except: [apply: 2] - - def apply_to_all(base_directory, namespace_app) do - base_directory - |> find_app_files() - |> tap(fn app_files -> - Mix.Shell.IO.info("Rewriting #{length(app_files)} app files") - end) - |> Enum.each(fn f -> apply(f, namespace_app) end) - end - - def apply(file_path, namespace_app) do - with {:ok, app_definition} <- Transform.Erlang.path_to_term(file_path), - {:ok, converted} <- convert(app_definition, namespace_app), - :ok <- File.write(file_path, converted) do - app_name = - file_path - |> Path.basename() - |> Path.rootname() - |> String.to_atom() - - if namespace_app do - namespaced_app_name = Namespace.Module.apply(app_name) - new_filename = "#{namespaced_app_name}.app" - - new_file_path = - file_path - |> Path.dirname() - |> Path.join(new_filename) - - File.rename!(file_path, new_file_path) - end - end - end - - defp find_app_files(base_directory) do - app_files_glob = Enum.join(Namespace.app_names(), ",") - - [base_directory, "**", "{#{app_files_glob}}.app"] - |> Path.join() - |> Path.wildcard() - end - - defp convert(app_definition, namespace_app) do - erlang_terms = - app_definition - |> visit(namespace_app) - |> Transform.Erlang.term_to_string() - - {:ok, erlang_terms} - end - - defp visit({:application, app_name, keys}, namespace_app) do - app = - if namespace_app do - Namespace.Module.apply(app_name) - else - app_name - end - - {:application, app, Enum.map(keys, fn k -> visit(k, namespace_app) end)} - end - - defp visit({:applications, app_list} = original, namespace_app) do - if namespace_app do - {:applications, Enum.map(app_list, &Namespace.Module.apply/1)} - else - original - end - end - - defp visit({:modules, module_list}, _) do - {:modules, Enum.map(module_list, &Namespace.Module.apply/1)} - end - - defp visit({:description, desc}, _) do - {:description, desc ++ ~c" namespaced by expert."} - end - - defp visit({:mod, {module_name, args}}, _) do - {:mod, {Namespace.Module.apply(module_name), args}} - end - - defp visit(key_value, _) do - key_value - end -end diff --git a/namespace/lib/mix/tasks/namespace/transform/configs.ex b/namespace/lib/mix/tasks/namespace/transform/configs.ex deleted file mode 100644 index 2c455a1f..00000000 --- a/namespace/lib/mix/tasks/namespace/transform/configs.ex +++ /dev/null @@ -1,41 +0,0 @@ -defmodule Mix.Tasks.Namespace.Transform.Configs do - alias Mix.Tasks.Namespace - - def apply_to_all(base_directory) do - base_directory - |> Path.join("**") - |> Path.wildcard() - |> Enum.map(&Path.absname/1) - |> tap(fn paths -> - Mix.Shell.IO.info("Rewriting #{length(paths)} config scripts.") - end) - |> Enum.each(&apply/1) - end - - def apply(path) do - namespaced = - path - |> File.read!() - |> Code.string_to_quoted!() - |> Macro.postwalk(fn - {:__aliases__, meta, alias} -> - namespaced_alias = - alias - |> Module.concat() - |> Namespace.Module.apply() - |> Module.split() - |> Enum.map(&String.to_atom/1) - - {:__aliases__, meta, namespaced_alias} - - atom when is_atom(atom) -> - Namespace.Module.apply(atom) - - ast -> - ast - end) - |> Macro.to_string() - - File.write!(path, namespaced) - end -end diff --git a/namespace/lib/mix/tasks/namespace/transform/scripts.ex b/namespace/lib/mix/tasks/namespace/transform/scripts.ex deleted file mode 100644 index 37ea5271..00000000 --- a/namespace/lib/mix/tasks/namespace/transform/scripts.ex +++ /dev/null @@ -1,99 +0,0 @@ -defmodule Mix.Tasks.Namespace.Transform.Scripts do - @moduledoc """ - A transform that updates any module in .script and .rel files with namespaced versions - """ - - alias Mix.Tasks.Namespace - alias Mix.Tasks.Namespace.Transform - - def apply_to_all(base_directory) do - base_directory - |> find_scripts() - |> tap(fn script_files -> - Mix.Shell.IO.info("Rewriting #{length(script_files)} scripts") - end) - |> Enum.each(&apply/1) - end - - def apply(file_path) do - with {:ok, app_definition} <- Transform.Erlang.path_to_term(file_path), - {:ok, converted} <- convert(app_definition) do - File.write(file_path, converted) - end - end - - @script_names ~w(start.script start_clean.script expert.rel) - defp find_scripts(base_directory) do - scripts_glob = "{" <> Enum.join(@script_names, ",") <> "}" - - [base_directory, "releases", "**", scripts_glob] - |> Path.join() - |> Path.wildcard() - end - - defp convert(app_definition) do - converted = visit(app_definition) - erlang_terms = Transform.Erlang.term_to_string(converted) - - script = """ - %% coding: utf-8 - #{erlang_terms} - """ - - {:ok, script} - end - - # for .rel files - defp visit({:release, release_vsn, erts_vsn, app_versions}) do - fixed_apps = - Enum.map(app_versions, fn {app_name, version, start_type} -> - {Namespace.Module.apply(app_name), version, start_type} - end) - - {:release, release_vsn, erts_vsn, fixed_apps} - end - - defp visit({:script, script_vsn, keys}) do - {:script, script_vsn, Enum.map(keys, &visit/1)} - end - - defp visit({:primLoad, app_list}) do - {:primLoad, Enum.map(app_list, &Namespace.Module.apply/1)} - end - - defp visit({:path, paths}) do - {:path, Enum.map(paths, &Namespace.Path.apply/1)} - end - - defp visit({:apply, {:application, :load, load_apps}}) do - {:apply, {:application, :load, Enum.map(load_apps, &visit/1)}} - end - - defp visit({:apply, {:application, :start_boot, apps_to_start}}) do - {:apply, {:application, :start_boot, Enum.map(apps_to_start, &Namespace.Module.apply/1)}} - end - - defp visit({:application, app_name, app_keys}) do - {:application, Namespace.Module.apply(app_name), Enum.map(app_keys, &visit/1)} - end - - defp visit({:application, app_name}) do - {:application, Namespace.Module.apply(app_name)} - end - - defp visit({:mod, {module_name, args}}) do - {:mod, {Namespace.Module.apply(module_name), Enum.map(args, &visit/1)}} - end - - defp visit({:modules, module_list}) do - {:modules, Enum.map(module_list, &Namespace.Module.apply/1)} - end - - defp visit({:applications, app_names}) do - {:applications, Enum.map(app_names, &Namespace.Module.apply/1)} - end - - defp visit(key_value) do - key_value - end -end diff --git a/namespace/lib/mix/tasks/namespace/abstract.ex b/namespace/lib/namespace/abstract.ex similarity index 53% rename from namespace/lib/mix/tasks/namespace/abstract.ex rename to namespace/lib/namespace/abstract.ex index 7b0cf33f..956a46c2 100644 --- a/namespace/lib/mix/tasks/namespace/abstract.ex +++ b/namespace/lib/namespace/abstract.ex @@ -1,4 +1,4 @@ -defmodule Mix.Tasks.Namespace.Abstract do +defmodule Namespace.Abstract do @moduledoc """ Transformations from erlang abstract syntax @@ -6,82 +6,92 @@ defmodule Mix.Tasks.Namespace.Abstract do https://www.erlang.org/doc/apps/erts/absform.html """ - alias Mix.Tasks.Namespace - - def rewrite(abstract_format) when is_list(abstract_format) do - Enum.map(abstract_format, &rewrite/1) + def code_from(path) do + with {:ok, {_orig_module, code_parts}} <- :beam_lib.chunks(path, [:abstract_code]), + {:ok, {:raw_abstract_v1, forms}} <- Keyword.fetch(code_parts, :abstract_code) do + {:ok, forms} + else + _ -> + {:error, :not_found} + end end - def rewrite(abstract_format) do - do_rewrite(abstract_format) + def run(abstract_format, opts) when is_list(abstract_format) do + Task.async(fn -> + Process.put(:abstract_code_opts, opts) + Enum.map(abstract_format, fn af -> rewrite(af) end) + end) + |> Task.await() end - # 8.1 Module Declarations and Forms + defp rewrite(forms) when is_list(forms) do + Enum.map(forms, fn af -> rewrite(af) end) + end - defp do_rewrite({:attribute, anno, :export, exported_functions}) do + defp rewrite({:attribute, anno, :export, exported_functions}) do {:attribute, anno, :export, exported_functions} end - defp do_rewrite({:attribute, anno, :behaviour, module}) do + defp rewrite({:attribute, anno, :behaviour, module}) do {:attribute, anno, :behaviour, rewrite_module(module)} end - defp do_rewrite({:attribute, anno, :import, {module, funs}}) do + defp rewrite({:attribute, anno, :import, {module, funs}}) do {:attribute, anno, :import, {rewrite_module(module), rewrite(funs)}} end - defp do_rewrite({:attribute, anno, :module, mod}) do + defp rewrite({:attribute, anno, :module, mod}) do {:attribute, anno, :module, rewrite_module(mod)} end - defp do_rewrite({:attribute, anno, :__impl__, attrs}) do + defp rewrite({:attribute, anno, :__impl__, attrs}) do {:attribute, anno, :__impl__, rewrite(attrs)} end - defp do_rewrite({:function, anno, name, arity, clauses}) do + defp rewrite({:function, anno, name, arity, clauses}) do {:function, anno, name, arity, rewrite(clauses)} end - defp do_rewrite({:attribute, anno, spec, {{name, arity}, spec_clauses}}) do + defp rewrite({:attribute, anno, spec, {{name, arity}, spec_clauses}}) do {:attribute, anno, rewrite(spec), {{name, arity}, rewrite(spec_clauses)}} end - defp do_rewrite({:attribute, anno, :spec, {{mod, name, arity}, clauses}}) do + defp rewrite({:attribute, anno, :spec, {{mod, name, arity}, clauses}}) do {:attribute, anno, :spec, {{rewrite(mod), name, arity}, rewrite(clauses)}} end - defp do_rewrite({:attribute, anno, :record, {name, fields}}) do + defp rewrite({:attribute, anno, :record, {name, fields}}) do {:attribute, anno, :record, {rewrite_module(name), rewrite(fields)}} end - defp do_rewrite({:attribute, anno, type, {name, type_rep, clauses}}) do + defp rewrite({:attribute, anno, type, {name, type_rep, clauses}}) do {:attribute, anno, type, {name, rewrite(type_rep), rewrite(clauses)}} end - defp do_rewrite({:for, target}) do + defp rewrite({:for, target}) do # Protocol implementation {:for, rewrite_module(target)} end - defp do_rewrite({:protocol, protocol}) do + defp rewrite({:protocol, protocol}) do {:protocol, rewrite_module(protocol)} end # Record Fields - defp do_rewrite({:record_field, anno, repr}) do + defp rewrite({:record_field, anno, repr}) do {:record_field, anno, rewrite(repr)} end - defp do_rewrite({:record_field, anno, repr_1, repr_2}) do + defp rewrite({:record_field, anno, repr_1, repr_2}) do {:record_field, anno, rewrite(repr_1), rewrite(repr_2)} end - defp do_rewrite({:typed_record_field, {:record_field, anno, repr_1}, repr_2}) do + defp rewrite({:typed_record_field, {:record_field, anno, repr_1}, repr_2}) do {:typed_record_field, {:record_field, anno, rewrite(repr_1)}, rewrite(repr_2)} end - defp do_rewrite({:typed_record_field, {:record_field, anno, repr_a, repr_e}, repr_t}) do + defp rewrite({:typed_record_field, {:record_field, anno, repr_a, repr_e}, repr_t}) do {:typed_record_field, {:record_field, anno, rewrite(repr_a), rewrite(repr_e)}, rewrite(repr_t)} end @@ -90,163 +100,163 @@ defmodule Mix.Tasks.Namespace.Abstract do # 8.2 Atomic Literals # only rewrite atoms, since they might be modules - defp do_rewrite({:atom, anno, literal}) when literal != :expert do + defp rewrite({:atom, anno, literal}) do {:atom, anno, rewrite_module(literal)} end # 8.3 Patterns # ignore bitstraings, they can't contain modules - defp do_rewrite({:match, anno, lhs, rhs}) do + defp rewrite({:match, anno, lhs, rhs}) do {:match, anno, rewrite(lhs), rewrite(rhs)} end - defp do_rewrite({:cons, anno, head, tail}) do + defp rewrite({:cons, anno, head, tail}) do {:cons, anno, rewrite(head), rewrite(tail)} end - defp do_rewrite({:map, anno, matches}) do + defp rewrite({:map, anno, matches}) do {:map, anno, rewrite(matches)} end - defp do_rewrite({:op, anno, op, lhs, rhs}) do + defp rewrite({:op, anno, op, lhs, rhs}) do {:op, anno, op, rewrite(lhs), rewrite(rhs)} end - defp do_rewrite({:op, anno, op, pattern}) do + defp rewrite({:op, anno, op, pattern}) do {:op, anno, op, rewrite(pattern)} end - defp do_rewrite({:tuple, anno, patterns}) do + defp rewrite({:tuple, anno, patterns}) do {:tuple, anno, rewrite(patterns)} end - defp do_rewrite({:var, anno, atom}) do + defp rewrite({:var, anno, atom}) do {:var, anno, rewrite_module(atom)} end # 8.4 Expressions - defp do_rewrite({:bc, anno, rep_e0, qualifiers}) do + defp rewrite({:bc, anno, rep_e0, qualifiers}) do {:bc, anno, rewrite(rep_e0), rewrite(qualifiers)} end - defp do_rewrite({:bin, anno, bin_elements}) do + defp rewrite({:bin, anno, bin_elements}) do {:bin, anno, rewrite(bin_elements)} end - defp do_rewrite({:bin_element, anno, elem, size, type}) do + defp rewrite({:bin_element, anno, elem, size, type}) do {:bin_element, anno, rewrite(elem), size, type} end - defp do_rewrite({:block, anno, body}) do + defp rewrite({:block, anno, body}) do {:block, anno, rewrite(body)} end - defp do_rewrite({:case, anno, expression, clauses}) do + defp rewrite({:case, anno, expression, clauses}) do {:case, anno, rewrite(expression), rewrite(clauses)} end - defp do_rewrite({:catch, anno, expression}) do + defp rewrite({:catch, anno, expression}) do {:catch, anno, rewrite(expression)} end - defp do_rewrite({:fun, anno, {:function, name, arity}}) do + defp rewrite({:fun, anno, {:function, name, arity}}) do {:fun, anno, {:function, rewrite(name), arity}} end - defp do_rewrite({:fun, anno, {:function, module, name, arity}}) do + defp rewrite({:fun, anno, {:function, module, name, arity}}) do {:fun, anno, {:function, rewrite(module), rewrite(name), arity}} end - defp do_rewrite({:fun, anno, {:clauses, clauses}}) do + defp rewrite({:fun, anno, {:clauses, clauses}}) do {:fun, anno, {:clauses, rewrite(clauses)}} end - defp do_rewrite({:named_fun, anno, name, clauses}) do + defp rewrite({:named_fun, anno, name, clauses}) do {:named_fun, anno, rewrite(name), rewrite(clauses)} end - defp do_rewrite({:call, anno, {:remote, remote_anno, module, fn_name}, args}) do + defp rewrite({:call, anno, {:remote, remote_anno, module, fn_name}, args}) do {:call, anno, {:remote, remote_anno, rewrite(module), fn_name}, rewrite(args)} end - defp do_rewrite({:call, anno, name, args}) do + defp rewrite({:call, anno, name, args}) do {:call, anno, rewrite(name), rewrite(args)} end - defp do_rewrite({:if, anno, clauses}) do + defp rewrite({:if, anno, clauses}) do {:if, anno, rewrite(clauses)} end - defp do_rewrite({:lc, anno, expression, qualifiers}) do + defp rewrite({:lc, anno, expression, qualifiers}) do {:lc, anno, rewrite(expression), rewrite(qualifiers)} end - defp do_rewrite({:map, anno, expression, clauses}) do + defp rewrite({:map, anno, expression, clauses}) do {:map, anno, rewrite(expression), rewrite(clauses)} end - defp do_rewrite({:maybe_match, anno, lhs, rhs}) do + defp rewrite({:maybe_match, anno, lhs, rhs}) do {:maybe_match, anno, rewrite(lhs), rewrite(rhs)} end - defp do_rewrite({:maybe, anno, body}) do + defp rewrite({:maybe, anno, body}) do {:maybe, anno, rewrite(body)} end - defp do_rewrite({:maybe, anno, maybe_body, {:else, anno, else_clauses}}) do + defp rewrite({:maybe, anno, maybe_body, {:else, anno, else_clauses}}) do {:maybe, anno, rewrite(maybe_body), {:else, anno, rewrite(else_clauses)}} end - defp do_rewrite({:receive, anno, clauses}) do + defp rewrite({:receive, anno, clauses}) do {:receive, anno, rewrite(clauses)} end - defp do_rewrite({:receive, anno, cases, expression, body}) do + defp rewrite({:receive, anno, cases, expression, body}) do {:receive, anno, rewrite(cases), rewrite(expression), rewrite(body)} end - defp do_rewrite({:record, anno, name, fields}) do + defp rewrite({:record, anno, name, fields}) do {:record, anno, rewrite_module(name), rewrite(fields)} end - defp do_rewrite({:record_field, anno, record_name, field_name, record_field}) do + defp rewrite({:record_field, anno, record_name, field_name, record_field}) do {:record_field, anno, rewrite_module(record_name), field_name, record_field} end - defp do_rewrite({:try, anno, body, case_clauses, catch_clauses}) do + defp rewrite({:try, anno, body, case_clauses, catch_clauses}) do {:try, anno, rewrite(body), rewrite(case_clauses), rewrite(catch_clauses)} end - defp do_rewrite({:try, anno, body, case_clauses, catch_clauses, after_clauses}) do + defp rewrite({:try, anno, body, case_clauses, catch_clauses, after_clauses}) do {:try, anno, rewrite(body), rewrite(case_clauses), rewrite(catch_clauses), rewrite(after_clauses)} end # Qualifiers - defp do_rewrite({:generate, anno, lhs, rhs}) do + defp rewrite({:generate, anno, lhs, rhs}) do {:generate, anno, rewrite(lhs), rewrite(rhs)} end - defp do_rewrite({:b_generate, anno, lhs, rhs}) do + defp rewrite({:b_generate, anno, lhs, rhs}) do {:b_generate, anno, rewrite(lhs), rewrite(rhs)} end # Associations - defp do_rewrite({:map_field_assoc, anno, key, value}) do + defp rewrite({:map_field_assoc, anno, key, value}) do {:map_field_assoc, anno, rewrite(key), rewrite(value)} end - defp do_rewrite({:map_field_exact, anno, key, value}) do + defp rewrite({:map_field_exact, anno, key, value}) do {:map_field_exact, anno, rewrite(key), rewrite(value)} end # 8.5 Clauses - defp do_rewrite({:clause, anno, lhs, guards, rhs}) do + defp rewrite({:clause, anno, lhs, guards, rhs}) do {:clause, anno, rewrite(lhs), rewrite(guards), rewrite(rhs)} end @@ -254,32 +264,32 @@ defmodule Mix.Tasks.Namespace.Abstract do # Guards seem covered by above clauses # 8.7 Types - defp do_rewrite({:ann_type, anno, clauses}) do + defp rewrite({:ann_type, anno, clauses}) do {:ann_type, anno, rewrite(clauses)} end - defp do_rewrite({:type, anno, :fun, [{:type, type_anno, :any}, type]}) do + defp rewrite({:type, anno, :fun, [{:type, type_anno, :any}, type]}) do {:type, anno, :fun, [{:type, type_anno, :any}, rewrite(type)]} end - defp do_rewrite({:type, anno, :map, key_values}) do + defp rewrite({:type, anno, :map, key_values}) do {:type, anno, :map, rewrite(key_values)} end - defp do_rewrite({:type, anno, predefined_type, expressions}) do + defp rewrite({:type, anno, predefined_type, expressions}) do {:type, anno, rewrite(predefined_type), rewrite(expressions)} end - defp do_rewrite({:remote_type, anno, [module, name, expressions]}) do + defp rewrite({:remote_type, anno, [module, name, expressions]}) do {:remote_type, anno, [rewrite_module(module), name, rewrite(expressions)]} end - defp do_rewrite({:user_type, anno, name, types}) do + defp rewrite({:user_type, anno, name, types}) do {:user_type, anno, rewrite_module(name), rewrite(types)} end # Catch all - defp do_rewrite(other) do + defp rewrite(other) do other end @@ -292,6 +302,7 @@ defmodule Mix.Tasks.Namespace.Abstract do end defp rewrite_module(module) do - Namespace.Module.apply(module) + opts = Process.get(:abstract_code_opts) + Namespace.Module.run(module, opts) end end diff --git a/namespace/lib/mix/tasks/namespace/code.ex b/namespace/lib/namespace/code.ex similarity index 70% rename from namespace/lib/mix/tasks/namespace/code.ex rename to namespace/lib/namespace/code.ex index 60523a3d..13b7207d 100644 --- a/namespace/lib/mix/tasks/namespace/code.ex +++ b/namespace/lib/namespace/code.ex @@ -1,4 +1,4 @@ -defmodule Mix.Tasks.Namespace.Code do +defmodule Namespace.Code do def compile(forms) do :compile.forms(forms, [:return_errors, :debug_info]) end diff --git a/namespace/lib/mix/tasks/namespace/transform/erlang.ex b/namespace/lib/namespace/erlang.ex similarity index 93% rename from namespace/lib/mix/tasks/namespace/transform/erlang.ex rename to namespace/lib/namespace/erlang.ex index 77fa59c5..eb906324 100644 --- a/namespace/lib/mix/tasks/namespace/transform/erlang.ex +++ b/namespace/lib/namespace/erlang.ex @@ -1,4 +1,4 @@ -defmodule Mix.Tasks.Namespace.Transform.Erlang do +defmodule Namespace.Erlang do @moduledoc """ Utilities for reading and writing erlang terms from and to text """ diff --git a/namespace/lib/mix/tasks/namespace/module.ex b/namespace/lib/namespace/module.ex similarity index 80% rename from namespace/lib/mix/tasks/namespace/module.ex rename to namespace/lib/namespace/module.ex index cf7b5361..fc92ae4f 100644 --- a/namespace/lib/mix/tasks/namespace/module.ex +++ b/namespace/lib/namespace/module.ex @@ -1,20 +1,21 @@ -defmodule Mix.Tasks.Namespace.Module do - alias Mix.Tasks.Namespace - +defmodule Namespace.Module do @namespace_prefix "XP" - def apply(module_name) do + def run(module_name, opts) do + apps = Keyword.fetch!(opts, :apps) + roots = Keyword.fetch!(opts, :roots) + cond do prefixed?(module_name) -> module_name - module_name in Namespace.app_names() -> + opts[:do_apps] && module_name in apps -> :"xp_#{module_name}" true -> module_name |> Atom.to_string() - |> apply_namespace() + |> apply_namespace(roots) end end @@ -40,8 +41,8 @@ defmodule Mix.Tasks.Namespace.Module do def prefixed?(_), do: false - defp apply_namespace("Elixir." <> rest) do - Namespace.root_modules() + defp apply_namespace("Elixir." <> rest, roots) do + roots |> Enum.map(fn module -> module |> Module.split() |> List.first() end) |> Enum.reduce_while(rest, fn root_module, module -> if has_root_module?(root_module, module) do @@ -59,7 +60,7 @@ defmodule Mix.Tasks.Namespace.Module do |> Module.concat() end - defp apply_namespace(erlang_module) do + defp apply_namespace(erlang_module, _) do String.to_atom(erlang_module) end @@ -69,10 +70,6 @@ defmodule Mix.Tasks.Namespace.Module do String.contains?(candidate, append_trailing_period(root_module)) end - defp namespace("Engine") do - "#{@namespace_prefix}ert" - end - defp namespace(orig) do @namespace_prefix <> orig end diff --git a/namespace/lib/namespace/path.ex b/namespace/lib/namespace/path.ex new file mode 100644 index 00000000..02948c76 --- /dev/null +++ b/namespace/lib/namespace/path.ex @@ -0,0 +1,27 @@ +defmodule Namespace.Path do + def run(path, opts) when is_list(path) do + path + |> List.to_string() + |> run(opts) + |> String.to_charlist() + end + + def run(path, opts) when is_binary(path) do + apps = Keyword.fetch!(opts, :apps) + + path + |> Path.split() + |> Enum.map(fn path_component -> + Enum.reduce(apps, path_component, fn app_name, path -> + if path == Atom.to_string(app_name) do + app_name + |> Namespace.Module.run(opts) + |> Atom.to_string() + else + path + end + end) + end) + |> Path.join() + end +end diff --git a/namespace/lib/namespace/transform/app_directories.ex b/namespace/lib/namespace/transform/app_directories.ex new file mode 100644 index 00000000..14aa1c43 --- /dev/null +++ b/namespace/lib/namespace/transform/app_directories.ex @@ -0,0 +1,18 @@ +defmodule Namespace.Transform.AppDirectories do + def run_all(base_directory, opts) do + app_globs = Enum.join(opts[:apps], "*,") + + base_directory + |> Path.join("lib/{#{app_globs}*}") + |> Path.wildcard() + |> Enum.each(fn d -> run(d, opts) end) + end + + def run(app_path, opts) do + namespaced_app_path = Namespace.Path.run(app_path, opts) + + with {:ok, _} <- File.rm_rf(namespaced_app_path) do + File.rename!(app_path, namespaced_app_path) + end + end +end diff --git a/namespace/lib/namespace/transform/apps.ex b/namespace/lib/namespace/transform/apps.ex new file mode 100644 index 00000000..73bf4926 --- /dev/null +++ b/namespace/lib/namespace/transform/apps.ex @@ -0,0 +1,85 @@ +defmodule Namespace.Transform.Apps do + @moduledoc """ + Applies namespacing to all modules defined in .app files + """ + + def run_all(base_directory, namespace_app, opts) do + app_files_glob = Enum.join(opts[:apps], ",") + + base_directory + |> Path.join("**/{#{app_files_glob}}.app") + |> Path.wildcard() + |> tap(fn app_files -> + Mix.Shell.IO.info("Rewriting #{length(app_files)} app files") + end) + |> Enum.each(fn f -> run(f, namespace_app, opts) end) + end + + def run(file_path, namespace_app, opts) do + with {:ok, app_definition} <- Namespace.Erlang.path_to_term(file_path), + {:ok, converted} <- convert(app_definition, namespace_app, opts), + :ok <- File.write(file_path, converted) do + app_name = + file_path + |> Path.basename() + |> Path.rootname() + |> String.to_atom() + + if namespace_app do + namespaced_app_name = Namespace.Module.run(app_name, opts) + new_filename = "#{namespaced_app_name}.app" + + new_file_path = + file_path + |> Path.dirname() + |> Path.join(new_filename) + + File.rename!(file_path, new_file_path) + end + end + end + + defp convert(app_definition, namespace_app, opts) do + erlang_terms = + app_definition + |> visit(namespace_app, opts) + |> Namespace.Erlang.term_to_string() + + {:ok, erlang_terms} + end + + defp visit({:application, app_name, keys}, namespace_app, opts) do + app = + if namespace_app do + Namespace.Module.run(app_name, opts) + else + app_name + end + + {:application, app, Enum.map(keys, fn k -> visit(k, namespace_app, opts) end)} + end + + defp visit({:applications, app_list} = original, namespace_app, opts) do + if namespace_app do + {:applications, Enum.map(app_list, fn app -> Namespace.Module.run(app, opts) end)} + else + original + end + end + + defp visit({:modules, module_list}, _, opts) do + {:modules, Enum.map(module_list, fn app -> Namespace.Module.run(app, opts) end)} + end + + defp visit({:description, desc}, _, _opts) do + {:description, desc ++ ~c" namespaced by expert."} + end + + defp visit({:mod, {module_name, args}}, _, opts) do + {:mod, {Namespace.Module.run(module_name, opts), args}} + end + + defp visit(key_value, _, _) do + key_value + end +end diff --git a/namespace/lib/mix/tasks/namespace/transform/beams.ex b/namespace/lib/namespace/transform/beams.ex similarity index 80% rename from namespace/lib/mix/tasks/namespace/transform/beams.ex rename to namespace/lib/namespace/transform/beams.ex index 7f921d01..97ef371b 100644 --- a/namespace/lib/mix/tasks/namespace/transform/beams.ex +++ b/namespace/lib/namespace/transform/beams.ex @@ -1,16 +1,12 @@ -defmodule Mix.Tasks.Namespace.Transform.Beams do +defmodule Namespace.Transform.Beams do @moduledoc """ A transformer that finds and replaces any instance of a module in a .beam file """ - alias Mix.Tasks.Namespace - alias Mix.Tasks.Namespace.Abstract - alias Mix.Tasks.Namespace.Code - - def apply_to_all(base_directory) do + def run_all(base_directory, do_apps, opts) do Mix.Shell.IO.info("Rewriting .beam files") consolidated_beams = find_consolidated_beams(base_directory) - app_beams = find_app_beams(base_directory) + app_beams = find_app_beams(base_directory, opts[:apps]) Mix.Shell.IO.info(" Found #{length(consolidated_beams)} protocols") Mix.Shell.IO.info(" Found #{length(app_beams)} app beam files") @@ -22,20 +18,27 @@ defmodule Mix.Tasks.Namespace.Transform.Beams do all_beams |> Task.async_stream(fn beam -> - apply_and_update_progress(beam, me) + apply_and_update_progress(beam, me, do_apps, opts) end) |> Stream.run() block_until_done(0, total_files) end - def apply(path) do + defp apply_and_update_progress(beam_file, caller, do_apps, opts) do + run(beam_file, do_apps, opts) + send(caller, :progress) + end + + def run(path, do_apps, opts) do erlang_path = String.to_charlist(path) + Process.put(:do_apps, do_apps) + with {:ok, forms} <- abstract_code(erlang_path), - rewritten_forms = Abstract.rewrite(forms), + rewritten_forms = Namespace.Abstract.run(forms, opts), true <- changed?(forms, rewritten_forms), - {:ok, module_name, binary} <- Code.compile(rewritten_forms) do + {:ok, module_name, binary} <- Namespace.Code.compile(rewritten_forms) do write_module_beam(path, module_name, binary) end end @@ -60,19 +63,14 @@ defmodule Mix.Tasks.Namespace.Transform.Beams do block_until_done(current, max) end - defp apply_and_update_progress(beam_file, caller) do - apply(beam_file) - send(caller, :progress) - end - defp find_consolidated_beams(base_directory) do [base_directory, "**", "consolidated", "*.beam"] |> Path.join() |> Path.wildcard() end - defp find_app_beams(base_directory) do - namespaced_apps = Enum.join(Namespace.app_names(), ",") + defp find_app_beams(base_directory, apps) do + namespaced_apps = Enum.join(apps, ",") apps_glob = "{#{namespaced_apps}}" [base_directory, "lib", apps_glob, "ebin/**", "*.beam"] diff --git a/namespace/lib/mix/tasks/namespace/transform/boots.ex b/namespace/lib/namespace/transform/boots.ex similarity index 76% rename from namespace/lib/mix/tasks/namespace/transform/boots.ex rename to namespace/lib/namespace/transform/boots.ex index 0fcc0c6e..0572e6fb 100644 --- a/namespace/lib/mix/tasks/namespace/transform/boots.ex +++ b/namespace/lib/namespace/transform/boots.ex @@ -1,17 +1,17 @@ -defmodule Mix.Tasks.Namespace.Transform.Boots do +defmodule Namespace.Transform.Boots do @moduledoc """ A transformer that re-builds .boot files by converting a .script file """ - def apply_to_all(base_directory) do + def run_all(base_directory, opts \\ []) do base_directory |> find_boot_files() |> tap(fn boot_files -> Mix.Shell.IO.info("Rebuilding #{length(boot_files)} boot files") end) - |> Enum.each(&apply/1) + |> Enum.each(&run(&1, opts)) end - def apply(file_path) do + def run(file_path, _opts \\ []) do file_path |> Path.rootname() |> String.to_charlist() diff --git a/namespace/lib/namespace/transform/configs.ex b/namespace/lib/namespace/transform/configs.ex new file mode 100644 index 00000000..dd2e0b59 --- /dev/null +++ b/namespace/lib/namespace/transform/configs.ex @@ -0,0 +1,41 @@ +# defmodule Mix.Tasks.Namespace.Transform.Configs do +# alias Mix.Tasks.Namespace +# +# def apply_to_all(base_directory) do +# base_directory +# |> Path.join("**") +# |> Path.wildcard() +# |> Enum.map(&Path.absname/1) +# |> tap(fn paths -> +# Mix.Shell.IO.info("Rewriting #{length(paths)} config scripts.") +# end) +# |> Enum.each(&apply/1) +# end +# +# def apply(path) do +# namespaced = +# path +# |> File.read!() +# |> Code.string_to_quoted!() +# |> Macro.postwalk(fn +# {:__aliases__, meta, alias} -> +# namespaced_alias = +# alias +# |> Module.concat() +# |> Namespace.Module.apply() +# |> Module.split() +# |> Enum.map(&String.to_atom/1) +# +# {:__aliases__, meta, namespaced_alias} +# +# atom when is_atom(atom) -> +# Namespace.Module.apply(atom) +# +# ast -> +# ast +# end) +# |> Macro.to_string() +# +# File.write!(path, namespaced) +# end +# end diff --git a/namespace/lib/namespace/transform/scripts.ex b/namespace/lib/namespace/transform/scripts.ex new file mode 100644 index 00000000..d741368c --- /dev/null +++ b/namespace/lib/namespace/transform/scripts.ex @@ -0,0 +1,96 @@ +# defmodule Mix.Tasks.Namespace.Transform.Scripts do +# @moduledoc """ +# A transform that updates any module in .script and .rel files with namespaced versions +# """ +# +# def apply_to_all(base_directory) do +# base_directory +# |> find_scripts() +# |> tap(fn script_files -> +# Mix.Shell.IO.info("Rewriting #{length(script_files)} scripts") +# end) +# |> Enum.each(&run/1) +# end +# +# def run(file_path) do +# with {:ok, app_definition} <- Namespace.Erlang.path_to_term(file_path), +# {:ok, converted} <- convert(app_definition) do +# File.write(file_path, converted) +# end +# end +# +# @script_names ~w(start.script start_clean.script expert.rel) +# defp find_scripts(base_directory) do +# scripts_glob = "{" <> Enum.join(@script_names, ",") <> "}" +# +# [base_directory, "releases", "**", scripts_glob] +# |> Path.join() +# |> Path.wildcard() +# end +# +# defp convert(app_definition) do +# converted = visit(app_definition) +# erlang_terms = Namespace.Erlang.term_to_string(converted) +# +# script = """ +# %% coding: utf-8 +# #{erlang_terms} +# """ +# +# {:ok, script} +# end +# +# # for .rel files +# defp visit({:release, release_vsn, erts_vsn, app_versions}) do +# fixed_apps = +# Enum.map(app_versions, fn {app_name, version, start_type} -> +# {Namespace.Module.run(app_name), version, start_type} +# end) +# +# {:release, release_vsn, erts_vsn, fixed_apps} +# end +# +# defp visit({:script, script_vsn, keys}) do +# {:script, script_vsn, Enum.map(keys, &visit/1)} +# end +# +# defp visit({:primLoad, app_list}) do +# {:primLoad, Enum.map(app_list, &Namespace.Module.run/1)} +# end +# +# defp visit({:path, paths}) do +# {:path, Enum.map(paths, &Namespace.Path.run/1)} +# end +# +# defp visit({:run, {:application, :load, load_apps}}) do +# {:run, {:application, :load, Enum.map(load_apps, &visit/1)}} +# end +# +# defp visit({:run, {:application, :start_boot, apps_to_start}}) do +# {:run, {:application, :start_boot, Enum.map(apps_to_start, &Namespace.Module.run/1)}} +# end +# +# defp visit({:application, app_name, app_keys}) do +# {:application, Namespace.Module.run(app_name), Enum.map(app_keys, &visit/1)} +# end +# +# defp visit({:application, app_name}) do +# {:application, Namespace.Module.run(app_name)} +# end +# +# defp visit({:mod, {module_name, args}}) do +# {:mod, {Namespace.Module.run(module_name), Enum.map(args, &visit/1)}} +# end +# +# defp visit({:modules, module_list}) do +# {:modules, Enum.map(module_list, &Namespace.Module.run/1)} +# end +# +# defp visit({:applications, app_names}) do +# {:applications, Enum.map(app_names, &Namespace.Module.run/1)} +# end +# +# defp visit(key_value) do +# key_value +# end +# end diff --git a/namespace/mix.exs b/namespace/mix.exs index 7ce796eb..0913247f 100644 --- a/namespace/mix.exs +++ b/namespace/mix.exs @@ -7,10 +7,19 @@ defmodule Namespace.MixProject do version: "0.1.0", elixir: "~> 1.17", start_permanent: Mix.env() == :prod, + elixirc_paths: elixirc_paths(Mix.env()), deps: deps() ] end + defp elixirc_paths(:test) do + ["lib", "test/support"] + end + + defp elixirc_paths(_) do + ["lib"] + end + # Run "mix help compile.app" to learn about applications. def application do [ diff --git a/namespace/mix.lock b/namespace/mix.lock new file mode 100644 index 00000000..0ac823b3 --- /dev/null +++ b/namespace/mix.lock @@ -0,0 +1,2 @@ +%{ +} diff --git a/namespace/test/namespace/abstract_test.exs b/namespace/test/namespace/abstract_test.exs new file mode 100644 index 00000000..b03a6974 --- /dev/null +++ b/namespace/test/namespace/abstract_test.exs @@ -0,0 +1,57 @@ +defmodule Namespace.AbstractTest do + use ExUnit.Case, async: true + + setup do + apps = [:foo, :bar, :baz] + roots = [Foo, Bar, Engine] + [apps: apps, roots: roots] + end + + test "rewrite module", %{apps: apps, roots: roots} do + {:ok, forms} = + Namespace.Abstract.code_from( + ~c"_build/test/lib/namespace/ebin/Elixir.Namespace.AbstractTest.Code.beam" + ) + + funcs = + forms + |> Namespace.Abstract.run(apps: apps, roots: roots) + |> Enum.filter(fn x -> match?({:function, _, _, _, _}, x) end) + |> Map.new(fn + {:function, _, name, arity, body} -> {{name, arity}, body} + _ -> nil + end) + + assert funcs[{:run, 0}] == [ + {:clause, 6, [], [], + [ + {:call, 7, {:atom, 7, :another}, []}, + {:call, 8, {:remote, 8, {:atom, 8, XPert}, {:atom, 8, :thing}}, []} + ]} + ] + + assert funcs[{:another, 0}] == [ + {:clause, 11, [], [], + [ + {:call, 12, {:remote, 12, {:atom, 12, Enum}, {:atom, 12, :map}}, + [ + {:call, 12, {:remote, 12, {:atom, 12, XPFoo}, {:atom, 12, :boo}}, []}, + {:fun, 12, + {:clauses, + [ + {:clause, 12, [{:var, 12, :_}], [], + [ + {:block, 0, + [ + {:call, 13, {:remote, 13, {:atom, 13, :xp_baz}, {:atom, 13, :run}}, + []}, + {:call, 14, {:remote, 14, {:atom, 14, XPBar.Foo}, {:atom, 14, :run}}, + [{:atom, 14, :xp_baz}]} + ]} + ]} + ]}} + ]} + ]} + ] + end +end diff --git a/namespace/test/namespace/module_test.exs b/namespace/test/namespace/module_test.exs new file mode 100644 index 00000000..312f024e --- /dev/null +++ b/namespace/test/namespace/module_test.exs @@ -0,0 +1,24 @@ +defmodule Namespace.ModuleTest do + use ExUnit.Case, async: true + + setup do + apps = [:foo, :bar, :baz] + roots = [Foo, Bar, Engine] + [apps: apps, roots: roots] + end + + test "namespaces a module", %{apps: apps, roots: roots} do + assert XPFoo == Namespace.Module.run(Foo, apps: apps, roots: roots) + assert :xp_baz == Namespace.Module.run(:baz, apps: apps, roots: roots) + assert XPert.Foo == Namespace.Module.run(Engine.Foo, apps: apps, roots: roots) + end + + test "doesn't namespace a module with a different root", %{apps: apps, roots: roots} do + refute XPFoo == Namespace.Module.run(Ding.Foo, apps: apps, roots: roots) + end + + test "doesnt namespace an already namespaced module", %{apps: apps, roots: roots} do + assert XPFoo == Namespace.Module.run(XPFoo, apps: apps, roots: roots) + assert :xp_baz == Namespace.Module.run(:xp_baz, apps: apps, roots: roots) + end +end diff --git a/namespace/test/namespace/path_test.exs b/namespace/test/namespace/path_test.exs new file mode 100644 index 00000000..fedceb47 --- /dev/null +++ b/namespace/test/namespace/path_test.exs @@ -0,0 +1,14 @@ +defmodule Namespace.PathTest do + use ExUnit.Case, async: true + + setup do + apps = [:foo, :bar, :baz] + roots = [Foo, Bar, Engine] + [apps: apps, roots: roots] + end + + test "namespaces charlist path", %{apps: apps, roots: roots} do + assert ~c"hello/xp_foo/ebin" == + Namespace.Path.run(~c"hello/foo/ebin", apps: apps, roots: roots) + end +end diff --git a/namespace/test/namespace/transform/app_directories_test.exs b/namespace/test/namespace/transform/app_directories_test.exs new file mode 100644 index 00000000..37900f23 --- /dev/null +++ b/namespace/test/namespace/transform/app_directories_test.exs @@ -0,0 +1,26 @@ +defmodule Namespace.Transform.AppDirectoriesTest do + use ExUnit.Case, async: true + + @moduletag tmp_dir: true + setup do + apps = [:foo, :bar, :baz] + roots = [Foo, Bar, Engine] + [apps: apps, roots: roots] + end + + test "renames the app directory", %{tmp_dir: dir, apps: apps, roots: roots} do + File.mkdir_p!(Path.join(dir, "lib/bar/ebin")) + File.mkdir_p!(Path.join(dir, "lib/foo/ebin")) + File.mkdir_p!(Path.join(dir, "lib/bob/ebin")) + + Namespace.Transform.AppDirectories.run_all(dir, apps: apps, roots: roots) + + refute File.exists?(Path.join(dir, "lib/bar/ebin/")) + assert File.exists?(Path.join(dir, "lib/xp_bar/ebin/")) + assert File.exists?(Path.join(dir, "lib/xp_foo/ebin/")) + + # doesn't run on dirs for apps not listed + assert File.exists?(Path.join(dir, "lib/bob/ebin/")) + refute File.exists?(Path.join(dir, "lib/xp_bob/ebin/")) + end +end diff --git a/namespace/test/namespace/transform/apps_test.exs b/namespace/test/namespace/transform/apps_test.exs new file mode 100644 index 00000000..964090f1 --- /dev/null +++ b/namespace/test/namespace/transform/apps_test.exs @@ -0,0 +1,62 @@ +defmodule Namespace.Transform.AppsTest do + use ExUnit.Case, async: true + + @app """ + {application,some_app, + [{config_mtime,1727544388}, + {optional_applications,[]}, + {applications,[kernel,stdlib,elixir,logger,sasl]}, + {description,"namespace"}, + {modules,['Elixir.SomeApp.Alice', + 'Elixir.SomeApp.Bob', + 'Elixir.AnotherApp.Foo', + 'Elixir.SomeApp.Carol']}, + {registered,[]}, + {vsn,"0.1.0"}]}. + """ + @moduletag tmp_dir: true + setup %{tmp_dir: dir} do + apps = [:some_app] + roots = [SomeApp] + path = Path.join(dir, "some/folder/path") + File.mkdir_p!(path) + File.write!(Path.join(path, "some_app.app"), @app) + [apps: apps, roots: roots, path: path] + end + + test "namespaces .app files", %{tmp_dir: dir, apps: apps, roots: roots} do + Namespace.Transform.Apps.run_all(dir, true, apps: apps, roots: roots) + + assert """ + {application,xp_some_app, + [{config_mtime,1727544388}, + {optional_applications,[]}, + {applications,[kernel,stdlib,elixir,logger,sasl]}, + {description,"namespace namespaced by expert."}, + {modules,['Elixir.XPSomeApp.Alice','Elixir.XPSomeApp.Bob', + 'Elixir.AnotherApp.Foo','Elixir.XPSomeApp.Carol']}, + {registered,[]}, + {vsn,"0.1.0"}]}. + """ == File.read!(Path.join(dir, "some/folder/path/xp_some_app.app")) + end + + test "doesn't namespace the actual app, only the modules", %{ + tmp_dir: dir, + apps: apps, + roots: roots + } do + Namespace.Transform.Apps.run_all(dir, false, apps: apps, roots: roots) + + assert """ + {application,some_app, + [{config_mtime,1727544388}, + {optional_applications,[]}, + {applications,[kernel,stdlib,elixir,logger,sasl]}, + {description,"namespace namespaced by expert."}, + {modules,['Elixir.XPSomeApp.Alice','Elixir.XPSomeApp.Bob', + 'Elixir.AnotherApp.Foo','Elixir.XPSomeApp.Carol']}, + {registered,[]}, + {vsn,"0.1.0"}]}. + """ == File.read!(Path.join(dir, "some/folder/path/some_app.app")) + end +end diff --git a/namespace/test/namespace/transform/beams_test.exs b/namespace/test/namespace/transform/beams_test.exs new file mode 100644 index 00000000..0e88fd81 --- /dev/null +++ b/namespace/test/namespace/transform/beams_test.exs @@ -0,0 +1,96 @@ +defmodule Namespace.Transform.BeamsTest do + use ExUnit.Case, async: true + @moduletag tmp_dir: true + setup %{tmp_dir: dir} do + apps = [:some_app, :bar] + roots = [SomeApp, Engine, Bar] + path = Path.join(dir, "lib/some_app/ebin") + File.mkdir_p!(path) + [apps: apps, roots: roots, path: path] + end + + test "rewrites the abstract code in the beam file", %{ + tmp_dir: dir, + apps: apps, + roots: roots, + path: path + } do + File.cp!( + "_build/test/lib/namespace/ebin/Elixir.SomeApp.beam", + Path.join(path, "Elixir.SomeApp.beam") + ) + + Namespace.Transform.Beams.run_all(dir, true, apps: apps, roots: roots) + + assert File.exists?(Path.join(path, "Elixir.XPSomeApp.beam")) + + {:ok, funcs} = + Namespace.Abstract.code_from( + Path.join(path, "Elixir.XPSomeApp.beam") + |> String.to_charlist() + ) + + funcs = + funcs + |> Enum.filter(fn x -> match?({:function, _, _, _, _}, x) end) + |> Map.new(fn + {:function, _, name, arity, body} -> {{name, arity}, body} + _ -> nil + end) + + assert funcs[{:run, 0}] == [ + {:clause, 24, [], [], + [ + {:call, 25, {:atom, 25, :another}, []}, + {:call, 26, {:remote, 26, {:atom, 26, XPert}, {:atom, 26, :thing}}, []} + ]} + ] + + assert funcs[{:another, 0}] == [ + { + :clause, + 29, + [], + [], + [ + { + :call, + 30, + {:remote, 30, {:atom, 30, Enum}, {:atom, 30, :map}}, + [ + {:call, 30, {:remote, 30, {:atom, 30, Foo}, {:atom, 30, :boo}}, []}, + { + :fun, + 30, + { + :clauses, + [ + { + :clause, + 30, + [{:var, 30, :_}], + [], + [ + { + :block, + 0, + [ + {:call, 31, + {:remote, 31, {:atom, 31, :baz}, {:atom, 31, :run}}, []}, + {:call, 32, + {:remote, 32, {:atom, 32, XPBar.Foo}, {:atom, 32, :run}}, + [{:atom, 32, :baz}]} + ] + } + ] + } + ] + } + } + ] + } + ] + } + ] + end +end diff --git a/namespace/test/namespace/transform/boots_test.exs b/namespace/test/namespace/transform/boots_test.exs new file mode 100644 index 00000000..4dfb6ba3 --- /dev/null +++ b/namespace/test/namespace/transform/boots_test.exs @@ -0,0 +1,6 @@ +defmodule Namespace.Transform.BootsTest do + use ExUnit.Case, async: true + + @tag skip: true + test "todo" +end diff --git a/namespace/test/namespace_test.exs b/namespace/test/namespace_test.exs deleted file mode 100644 index 0a9c1510..00000000 --- a/namespace/test/namespace_test.exs +++ /dev/null @@ -1,8 +0,0 @@ -defmodule NamespaceTest do - use ExUnit.Case - doctest Namespace - - test "greets the world" do - assert Namespace.hello() == :world - end -end diff --git a/namespace/test/support/fixtures/forms.ex b/namespace/test/support/fixtures/forms.ex new file mode 100644 index 00000000..1d672c0c --- /dev/null +++ b/namespace/test/support/fixtures/forms.ex @@ -0,0 +1,35 @@ +defmodule Namespace.AbstractTest.Code do + @compile {:no_warn_undefined, :baz} + @compile {:no_warn_undefined, Bar.Foo} + @compile {:no_warn_undefined, Engine} + @compile {:no_warn_undefined, Foo} + def run do + another() + Engine.thing() + end + + defp another() do + for _ <- Foo.boo() do + :baz.run() + Bar.Foo.run(:baz) + end + end +end + +defmodule SomeApp do + @compile {:no_warn_undefined, :baz} + @compile {:no_warn_undefined, Bar.Foo} + @compile {:no_warn_undefined, Engine} + @compile {:no_warn_undefined, Foo} + def run do + another() + Engine.thing() + end + + defp another() do + for _ <- Foo.boo() do + :baz.run() + Bar.Foo.run(:baz) + end + end +end diff --git a/namespace/test/test_helper.exs b/namespace/test/test_helper.exs index 869559e7..436e9d2e 100644 --- a/namespace/test/test_helper.exs +++ b/namespace/test/test_helper.exs @@ -1 +1 @@ -ExUnit.start() +ExUnit.start(exclude: [skip: true]) From 6a5c0ea0c11a7db1e9fb48754d46af9cd309cee5 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Fri, 25 Oct 2024 16:20:17 -0400 Subject: [PATCH 04/18] spike(expert): don't namespace unnecesasry apps/roots The release was not working correctly, as the mix.exs file was not namespaced, so the call to Burrito code was not transformed. Burrito is not code that needs to be namespaced, so we exclude it (along with Jason). --- .gitignore | 1 + expert/lib/expert/release.ex | 15 --------------- expert/mix.exs | 9 +-------- justfile | 30 +++++++++++++++++++++++++----- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index e81695ec..5618d376 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .expert-lsp/ +erl_crash.dump diff --git a/expert/lib/expert/release.ex b/expert/lib/expert/release.ex index 5d1dd046..c5df2853 100644 --- a/expert/lib/expert/release.ex +++ b/expert/lib/expert/release.ex @@ -2,21 +2,6 @@ defmodule Expert.Release do def assemble(release) do engine_path = Path.expand("../../../engine", __DIR__) - {_, 0} = - System.cmd("mix", ["build"], - cd: engine_path, - env: [ - {"MIX_ENV", to_string(Mix.env())} - ] - ) - - {_, 0} = - System.cmd("mix", ["namespace"], - env: [ - {"MIX_ENV", to_string(Mix.env())} - ] - ) - source = Path.join([engine_path, "_build/#{Mix.env()}"]) dest = diff --git a/expert/mix.exs b/expert/mix.exs index b89b4226..feeb947a 100644 --- a/expert/mix.exs +++ b/expert/mix.exs @@ -9,7 +9,6 @@ defmodule Expert.MixProject do start_permanent: Mix.env() == :prod, elixirc_paths: elixirc_paths(Mix.env()), releases: releases(), - aliases: aliases(), default_release: :expert, deps: deps() ] @@ -44,12 +43,6 @@ defmodule Expert.MixProject do defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] - defp aliases() do - [ - namespace: "namespace" - ] - end - # Run "mix help deps" to learn about dependencies. defp deps do [ @@ -57,7 +50,7 @@ defmodule Expert.MixProject do github: "elixir-tools/gen_lsp", branch: "change-schematic-function", override: true}, # {:gen_lsp, "~> 0.10"}, {:burrito, "~> 1.0", only: [:dev, :prod]}, - {:namespace, path: "../namespace", only: [:dev]} + {:namespace, path: "../namespace"} ] end end diff --git a/justfile b/justfile index 5e254342..7684be98 100644 --- a/justfile +++ b/justfile @@ -3,7 +3,14 @@ deps project: cd {{project}} mix deps.get -build project: +run project +ARGS: + #!/usr/bin/env bash + set -euo pipefail + cd {{project}} + eval "{{ ARGS }}" + + +compile project: #!/usr/bin/env bash set -euo pipefail @@ -23,10 +30,21 @@ build project: rm -rf "$safe_dir/" # copy new code in the safe area cp -a "$build_dir/." "$safe_dir/" + +build project *args: (compile project) + #!/usr/bin/env bash + set -euo pipefail + mix_env="${MIX_ENV:-dev}" + build_dir="_build/$mix_env" + + cd {{project}} + # namespace the new code - mix namespace --directory "$build_dir" + mix namespace --directory "$build_dir" {{args}} -start *opts="--port 9000": (build "engine") (build "expert") +start *opts="--port 9000": \ + (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --apps") \ + (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine") #!/usr/bin/env bash cd expert @@ -48,7 +66,9 @@ format project: mix format [unix] -release-local: +release-local: \ + (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --apps") \ + (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine") #!/usr/bin/env bash cd expert case "{{os()}}-{{arch()}}" in @@ -65,7 +85,7 @@ release-local: exit 1;; esac - EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="$target" MIX_ENV=prod mix release + EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="$target" MIX_ENV=prod mix release --no-compile [windows] release-local: From dd859656b26ff47ac1cfe179760c9be52005101c Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Fri, 25 Oct 2024 16:20:33 -0400 Subject: [PATCH 05/18] spike(expert): correctly report compiler errors --- expert/lib/expert.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expert/lib/expert.ex b/expert/lib/expert.ex index 5d9ee741..a90d3742 100644 --- a/expert/lib/expert.ex +++ b/expert/lib/expert.ex @@ -145,7 +145,7 @@ defmodule Expert do def handle_info({:compiler_result, _name, result}, lsp) do case result do - {status, diagnostics} when status in [:ok, :noop] -> + {status, diagnostics} when status not in [:ok, :noop] -> per_file = for d <- diagnostics, reduce: Map.new() do acc -> From 00ef0bcd5bde254efa75e8b2096bd770c507089e Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 10:47:56 -0400 Subject: [PATCH 06/18] spike(namespace): fix tests --- namespace/lib/mix/tasks/namespace.ex | 17 +++++++---------- namespace/lib/namespace/module.ex | 11 +++++++++-- namespace/lib/namespace/transform/apps.ex | 10 ++++++---- namespace/lib/namespace/transform/beams.ex | 11 ++++++----- namespace/test/namespace/abstract_test.exs | 7 +++---- namespace/test/namespace/module_test.exs | 18 +++++++++++++----- namespace/test/namespace/path_test.exs | 7 ++++++- .../transform/app_directories_test.exs | 2 +- .../test/namespace/transform/apps_test.exs | 15 +++++++++++++-- .../test/namespace/transform/beams_test.exs | 14 ++++++++++++-- 10 files changed, 76 insertions(+), 36 deletions(-) diff --git a/namespace/lib/mix/tasks/namespace.ex b/namespace/lib/mix/tasks/namespace.ex index be55e7ad..732b303a 100644 --- a/namespace/lib/mix/tasks/namespace.ex +++ b/namespace/lib/mix/tasks/namespace.ex @@ -19,7 +19,7 @@ defmodule Mix.Tasks.Namespace do OptionParser.parse!(argv, strict: [ directory: :string, - apps: :boolean, + dot_apps: :boolean, include_app: :keep, include_root: :keep, exclude_app: :keep, @@ -27,28 +27,25 @@ defmodule Mix.Tasks.Namespace do ] ) - base_directory = options[:directory] + base_directory = Keyword.fetch!(options, :directory) include_apps = Keyword.get_values(options, :include_app) |> Enum.map(&String.to_atom/1) include_roots = Keyword.get_values(options, :include_root) |> Enum.map(&Module.concat([&1])) exclude_apps = Keyword.get_values(options, :exclude_app) |> Enum.map(&String.to_atom/1) exclude_roots = Keyword.get_values(options, :exclude_root) |> Enum.map(&Module.concat([&1])) + apps = Enum.uniq(Mix.Project.deps_apps() ++ include_apps) -- exclude_apps - apps = (Mix.Project.deps_apps() ++ include_apps) -- exclude_apps - - roots_from_apps = apps |> root_modules_for_apps() |> Map.values() |> List.flatten() |> Enum.uniq() + roots_from_apps = + apps |> root_modules_for_apps() |> Map.values() |> List.flatten() |> Enum.uniq() roots = (roots_from_apps ++ include_roots) -- exclude_roots - opts = [apps: apps, roots: roots, do_apps: options[:apps]] - Namespace.Transform.Apps.run_all(base_directory, options[:apps], opts) + Namespace.Transform.Apps.run_all(base_directory, opts) - Namespace.Transform.Beams.run_all(base_directory, options[:apps], opts) - # Transform.Configs.run_all(base_directory) - # consolidated + Namespace.Transform.Beams.run_all(base_directory, opts) if options[:apps] do Namespace.Transform.AppDirectories.run_all(base_directory, opts) diff --git a/namespace/lib/namespace/module.ex b/namespace/lib/namespace/module.ex index fc92ae4f..bbf033e7 100644 --- a/namespace/lib/namespace/module.ex +++ b/namespace/lib/namespace/module.ex @@ -43,6 +43,7 @@ defmodule Namespace.Module do defp apply_namespace("Elixir." <> rest, roots) do roots + |> Enum.filter(fn module -> Macro.classify_atom(module) == :alias end) |> Enum.map(fn module -> module |> Module.split() |> List.first() end) |> Enum.reduce_while(rest, fn root_module, module -> if has_root_module?(root_module, module) do @@ -60,8 +61,14 @@ defmodule Namespace.Module do |> Module.concat() end - defp apply_namespace(erlang_module, _) do - String.to_atom(erlang_module) + defp apply_namespace(erlang_module, roots) do + erlang_module = String.to_atom(erlang_module) + + if erlang_module in roots do + :"xp_#{erlang_module}" + else + erlang_module + end end defp has_root_module?(root_module, root_module), do: true diff --git a/namespace/lib/namespace/transform/apps.ex b/namespace/lib/namespace/transform/apps.ex index 73bf4926..b6ed2098 100644 --- a/namespace/lib/namespace/transform/apps.ex +++ b/namespace/lib/namespace/transform/apps.ex @@ -1,9 +1,9 @@ defmodule Namespace.Transform.Apps do @moduledoc """ - Applies namespacing to all modules defined in .app files + Namespaces modules and app names inside .app files. """ - def run_all(base_directory, namespace_app, opts) do + def run_all(base_directory, opts) do app_files_glob = Enum.join(opts[:apps], ",") base_directory @@ -12,10 +12,12 @@ defmodule Namespace.Transform.Apps do |> tap(fn app_files -> Mix.Shell.IO.info("Rewriting #{length(app_files)} app files") end) - |> Enum.each(fn f -> run(f, namespace_app, opts) end) + |> Enum.each(fn f -> run(f, opts) end) end - def run(file_path, namespace_app, opts) do + def run(file_path, opts) do + namespace_app = opts[:do_apps] + with {:ok, app_definition} <- Namespace.Erlang.path_to_term(file_path), {:ok, converted} <- convert(app_definition, namespace_app, opts), :ok <- File.write(file_path, converted) do diff --git a/namespace/lib/namespace/transform/beams.ex b/namespace/lib/namespace/transform/beams.ex index 97ef371b..9ee1036a 100644 --- a/namespace/lib/namespace/transform/beams.ex +++ b/namespace/lib/namespace/transform/beams.ex @@ -3,7 +3,7 @@ defmodule Namespace.Transform.Beams do A transformer that finds and replaces any instance of a module in a .beam file """ - def run_all(base_directory, do_apps, opts) do + def run_all(base_directory, opts) do Mix.Shell.IO.info("Rewriting .beam files") consolidated_beams = find_consolidated_beams(base_directory) app_beams = find_app_beams(base_directory, opts[:apps]) @@ -18,19 +18,20 @@ defmodule Namespace.Transform.Beams do all_beams |> Task.async_stream(fn beam -> - apply_and_update_progress(beam, me, do_apps, opts) + apply_and_update_progress(beam, me, opts) end) |> Stream.run() block_until_done(0, total_files) end - defp apply_and_update_progress(beam_file, caller, do_apps, opts) do - run(beam_file, do_apps, opts) + defp apply_and_update_progress(beam_file, caller, opts) do + run(beam_file, opts) send(caller, :progress) end - def run(path, do_apps, opts) do + def run(path, opts) do + do_apps = opts[:do_apps] erlang_path = String.to_charlist(path) Process.put(:do_apps, do_apps) diff --git a/namespace/test/namespace/abstract_test.exs b/namespace/test/namespace/abstract_test.exs index b03a6974..2256c676 100644 --- a/namespace/test/namespace/abstract_test.exs +++ b/namespace/test/namespace/abstract_test.exs @@ -26,7 +26,7 @@ defmodule Namespace.AbstractTest do {:clause, 6, [], [], [ {:call, 7, {:atom, 7, :another}, []}, - {:call, 8, {:remote, 8, {:atom, 8, XPert}, {:atom, 8, :thing}}, []} + {:call, 8, {:remote, 8, {:atom, 8, XPEngine}, {:atom, 8, :thing}}, []} ]} ] @@ -43,10 +43,9 @@ defmodule Namespace.AbstractTest do [ {:block, 0, [ - {:call, 13, {:remote, 13, {:atom, 13, :xp_baz}, {:atom, 13, :run}}, - []}, + {:call, 13, {:remote, 13, {:atom, 13, :baz}, {:atom, 13, :run}}, []}, {:call, 14, {:remote, 14, {:atom, 14, XPBar.Foo}, {:atom, 14, :run}}, - [{:atom, 14, :xp_baz}]} + [{:atom, 14, :baz}]} ]} ]} ]}} diff --git a/namespace/test/namespace/module_test.exs b/namespace/test/namespace/module_test.exs index 312f024e..7e65863b 100644 --- a/namespace/test/namespace/module_test.exs +++ b/namespace/test/namespace/module_test.exs @@ -3,22 +3,30 @@ defmodule Namespace.ModuleTest do setup do apps = [:foo, :bar, :baz] - roots = [Foo, Bar, Engine] + roots = [Foo, Bar, Engine, :something] [apps: apps, roots: roots] end test "namespaces a module", %{apps: apps, roots: roots} do assert XPFoo == Namespace.Module.run(Foo, apps: apps, roots: roots) - assert :xp_baz == Namespace.Module.run(:baz, apps: apps, roots: roots) - assert XPert.Foo == Namespace.Module.run(Engine.Foo, apps: apps, roots: roots) + assert XPEngine.Foo == Namespace.Module.run(Engine.Foo, apps: apps, roots: roots) end test "doesn't namespace a module with a different root", %{apps: apps, roots: roots} do refute XPFoo == Namespace.Module.run(Ding.Foo, apps: apps, roots: roots) end - test "doesnt namespace an already namespaced module", %{apps: apps, roots: roots} do + test "doesn't namespace an already namespaced module", %{apps: apps, roots: roots} do assert XPFoo == Namespace.Module.run(XPFoo, apps: apps, roots: roots) - assert :xp_baz == Namespace.Module.run(:xp_baz, apps: apps, roots: roots) + end + + test "namespaces app name if enabled", %{apps: apps, roots: roots} do + assert :xp_baz == Namespace.Module.run(:baz, do_apps: true, apps: apps, roots: roots) + assert :baz == Namespace.Module.run(:baz, do_apps: false, apps: apps, roots: roots) + end + + test "namespaces erlang module", %{apps: apps, roots: roots} do + assert :xp_something == + Namespace.Module.run(:something, do_apps: false, apps: apps, roots: roots) end end diff --git a/namespace/test/namespace/path_test.exs b/namespace/test/namespace/path_test.exs index fedceb47..8c34b45c 100644 --- a/namespace/test/namespace/path_test.exs +++ b/namespace/test/namespace/path_test.exs @@ -9,6 +9,11 @@ defmodule Namespace.PathTest do test "namespaces charlist path", %{apps: apps, roots: roots} do assert ~c"hello/xp_foo/ebin" == - Namespace.Path.run(~c"hello/foo/ebin", apps: apps, roots: roots) + Namespace.Path.run(~c"hello/foo/ebin", do_apps: true, apps: apps, roots: roots) + end + + test "doesn't namespace if do_apps is false", %{apps: apps, roots: roots} do + assert ~c"hello/foo/ebin" == + Namespace.Path.run(~c"hello/foo/ebin", do_apps: false, apps: apps, roots: roots) end end diff --git a/namespace/test/namespace/transform/app_directories_test.exs b/namespace/test/namespace/transform/app_directories_test.exs index 37900f23..5387fef0 100644 --- a/namespace/test/namespace/transform/app_directories_test.exs +++ b/namespace/test/namespace/transform/app_directories_test.exs @@ -13,7 +13,7 @@ defmodule Namespace.Transform.AppDirectoriesTest do File.mkdir_p!(Path.join(dir, "lib/foo/ebin")) File.mkdir_p!(Path.join(dir, "lib/bob/ebin")) - Namespace.Transform.AppDirectories.run_all(dir, apps: apps, roots: roots) + Namespace.Transform.AppDirectories.run_all(dir, do_apps: true, apps: apps, roots: roots) refute File.exists?(Path.join(dir, "lib/bar/ebin/")) assert File.exists?(Path.join(dir, "lib/xp_bar/ebin/")) diff --git a/namespace/test/namespace/transform/apps_test.exs b/namespace/test/namespace/transform/apps_test.exs index 964090f1..5eff3f47 100644 --- a/namespace/test/namespace/transform/apps_test.exs +++ b/namespace/test/namespace/transform/apps_test.exs @@ -1,5 +1,6 @@ defmodule Namespace.Transform.AppsTest do use ExUnit.Case, async: true + import ExUnit.CaptureIO @app """ {application,some_app, @@ -25,7 +26,12 @@ defmodule Namespace.Transform.AppsTest do end test "namespaces .app files", %{tmp_dir: dir, apps: apps, roots: roots} do - Namespace.Transform.Apps.run_all(dir, true, apps: apps, roots: roots) + {_, io} = + with_io(fn -> + Namespace.Transform.Apps.run_all(dir, do_apps: true, apps: apps, roots: roots) + end) + + assert io =~ "Rewriting 1 app files" assert """ {application,xp_some_app, @@ -45,7 +51,12 @@ defmodule Namespace.Transform.AppsTest do apps: apps, roots: roots } do - Namespace.Transform.Apps.run_all(dir, false, apps: apps, roots: roots) + {_, io} = + with_io(fn -> + Namespace.Transform.Apps.run_all(dir, do_apps: false, apps: apps, roots: roots) + end) + + assert io =~ "Rewriting 1 app files" assert """ {application,some_app, diff --git a/namespace/test/namespace/transform/beams_test.exs b/namespace/test/namespace/transform/beams_test.exs index 0e88fd81..16b63e1e 100644 --- a/namespace/test/namespace/transform/beams_test.exs +++ b/namespace/test/namespace/transform/beams_test.exs @@ -1,5 +1,7 @@ defmodule Namespace.Transform.BeamsTest do use ExUnit.Case, async: true + import ExUnit.CaptureIO + @moduletag tmp_dir: true setup %{tmp_dir: dir} do apps = [:some_app, :bar] @@ -20,7 +22,15 @@ defmodule Namespace.Transform.BeamsTest do Path.join(path, "Elixir.SomeApp.beam") ) - Namespace.Transform.Beams.run_all(dir, true, apps: apps, roots: roots) + {_, io} = + with_io(fn -> + Namespace.Transform.Beams.run_all(dir, do_apps: true, apps: apps, roots: roots) + end) + + assert io =~ "Rewriting .beam files" + assert io =~ "Found 1 app beam files" + assert io =~ "Applying namespace:" + assert io =~ "done" assert File.exists?(Path.join(path, "Elixir.XPSomeApp.beam")) @@ -42,7 +52,7 @@ defmodule Namespace.Transform.BeamsTest do {:clause, 24, [], [], [ {:call, 25, {:atom, 25, :another}, []}, - {:call, 26, {:remote, 26, {:atom, 26, XPert}, {:atom, 26, :thing}}, []} + {:call, 26, {:remote, 26, {:atom, 26, XPEngine}, {:atom, 26, :thing}}, []} ]} ] From fe66b7813212a9a8fb762133362be56d74a8108d Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 10:48:15 -0400 Subject: [PATCH 07/18] spike(just): improve justfile --- justfile | 61 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/justfile b/justfile index 7684be98..9a019a1f 100644 --- a/justfile +++ b/justfile @@ -1,3 +1,9 @@ +mix_env := env('MIX_ENV', 'dev') +build_dir := "_build" / mix_env +safe_dir := "_build" / mix_env + "safe" + +apps := "expert engine namespace" + deps project: #!/usr/bin/env bash cd {{project}} @@ -9,56 +15,57 @@ run project +ARGS: cd {{project}} eval "{{ ARGS }}" - compile project: #!/usr/bin/env bash set -euo pipefail cd {{project}} - mix_env="${MIX_ENV:-dev}" - build_dir="_build/$mix_env" - safe_dir="_build/$mix_env-safe" # create our safekeeping area - mkdir -p "$safe_dir" + mkdir -p {{safe_dir}} # delete what is currently in the build dir - rm -rf "$build_dir" + rm -rf {{ build_dir }} # move our build artifacts from safekeeping to the build area - cp -a "$safe_dir/." "$build_dir/" + cp -a "{{ safe_dir }}/." "{{ build_dir }}/" # compile the safe kept code, respects incremental compilation mix compile # prep the safe area for new code - rm -rf "$safe_dir/" + rm -rf "{{ safe_dir }}/" # copy new code in the safe area - cp -a "$build_dir/." "$safe_dir/" + cp -a "{{ build_dir }}/." "{{ safe_dir }}/" build project *args: (compile project) #!/usr/bin/env bash set -euo pipefail - mix_env="${MIX_ENV:-dev}" - build_dir="_build/$mix_env" cd {{project}} # namespace the new code - mix namespace --directory "$build_dir" {{args}} + mix namespace --directory "{{ build_dir }}" {{args}} + +build-engine: (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --dot-apps") +build-expert: (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine") -start *opts="--port 9000": \ - (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --apps") \ - (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine") +start *opts="--port 9000": build-engine build-expert #!/usr/bin/env bash cd expert # no compile is important so it doesn't mess up the namespacing - EXPERT_ENGINE_PATH="../engine/_build/${MIX_ENV:-dev}/" mix run \ + EXPERT_ENGINE_PATH="../engine/_build/{{ mix_env }}/" mix run \ --no-compile \ --no-halt \ -e "Application.ensure_all_started(:expert)" \ -- {{opts}} -test project: +test: #!/usr/bin/env bash - cd {{project}} - mix test + + for project in {{ apps }}; do + ( + cd "$project" + + mix test + ) + done format project: #!/usr/bin/env bash @@ -66,9 +73,7 @@ format project: mix format [unix] -release-local: \ - (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --apps") \ - (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine") +release-local: build-engine build-expert #!/usr/bin/env bash cd expert case "{{os()}}-{{arch()}}" in @@ -88,14 +93,14 @@ release-local: \ EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="$target" MIX_ENV=prod mix release --no-compile [windows] -release-local: +release-local: build-engine build-expert # idk actually how to set env vars like this on windows, might crash - EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release + EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release --no-compile -release-all: +release-all: build-engine build-expert cd expert - EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release + EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release --no-compile -release-plain: +release-plain: build-engine build-expert cd expert - MIX_ENV=prod mix release plain + MIX_ENV=prod mix release plain --no-compile From 176b8030df1c3f11396cacea3607134c17969056 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 10:49:06 -0400 Subject: [PATCH 08/18] spike(namespace): improve docs --- namespace/lib/mix/tasks/namespace.ex | 31 +++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/namespace/lib/mix/tasks/namespace.ex b/namespace/lib/mix/tasks/namespace.ex index 732b303a..9c65e88b 100644 --- a/namespace/lib/mix/tasks/namespace.ex +++ b/namespace/lib/mix/tasks/namespace.ex @@ -1,14 +1,31 @@ defmodule Mix.Tasks.Namespace do @moduledoc """ - This task is used after a release is assembled, and investigates the remote_control - app for its dependencies, at which point it applies transformers to various parts of the - app. + This task will apply namespacing to a set of .beam and .app files in the given directory. - Transformers take a path, find their relevant files and apply transforms to them. For example, - the Beams transformer will find any instances of modules in .beam files, and will apply namepaces - to them if the module is one of the modules defined in a dependency. + Primarily works on a list of application and a list of "module roots". - This task takes a single argument, which is the full path to the release. + A module root is the first segment of an Elixir module, e.g., "Foo" in "Foo.Bar.Baz". + + The initial list of apps and roots (before additional inclusions and exclusions) are derived from + fetching the projects deps via `Mix.Project.deps_apps/0`. From there, each dependency's modules are + fetched via `:application.get_key(dep_app, :modules)`. + + ## Options + + * `--directory` - The active working directory (required) + * `--[no-]dot-apps` - Whether to namespace application names and .app files at all. Useful to disable if you dont need to start the project like a normal application. Defaults to false. + * `--include-app` - Adds the given application to the list of applications to namespace. + * `--exclude-app` - Removes the given application from the list of applications to namespace. + * `--include-root` - Adds the given module "root" to the list of "roots" to namespace. + * `--exclude-root` - Removes the given module "root" from the list of "roots" to namespace. + + + ## Usage + + ```bash + mix namespace --directory _build/prod --include-app engine --include-root Engine --exclude-app namespace --dot-apps + mix namespace --directory _build/dev --include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine + ``` """ use Mix.Task From 3363cd339ab4452749fe3927173467e14d756b46 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 10:49:14 -0400 Subject: [PATCH 09/18] spike(engine): remove aliases --- engine/mix.exs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/engine/mix.exs b/engine/mix.exs index 1b18b36a..bf0a3e9c 100644 --- a/engine/mix.exs +++ b/engine/mix.exs @@ -7,14 +7,6 @@ defmodule Engine.MixProject do version: "0.1.0", elixir: "~> 1.17", start_permanent: Mix.env() == :prod, - aliases: [ - namespace: "namespace --apps", - build: [ - "cmd rm -rf _build/#{Mix.env()}", - "compile", - "namespace --apps --directory _build/#{Mix.env()}" - ] - ], deps: deps() ] end From 7a3f8cc369ccf6b1f2a39db3d5636dd43ab645d2 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 11:18:27 -0400 Subject: [PATCH 10/18] spike(just): fix release-all Further excludes some apps from namespacing so burrito can download the requisite OTPs --- justfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 9a019a1f..729bfdeb 100644 --- a/justfile +++ b/justfile @@ -43,7 +43,7 @@ build project *args: (compile project) mix namespace --directory "{{ build_dir }}" {{args}} build-engine: (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --dot-apps") -build-expert: (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app namespace --exclude-root Jason --include-root Engine") +build-expert: (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app req --exclude-app finch --exclude-app nimble_options --exclude-app nimble_pool --exclude-app namespace --exclude-root Jason --include-root Engine") start *opts="--port 9000": build-engine build-expert #!/usr/bin/env bash @@ -98,9 +98,11 @@ release-local: build-engine build-expert EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release --no-compile release-all: build-engine build-expert + #!/usr/bin/env bash cd expert EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release --no-compile release-plain: build-engine build-expert + #!/usr/bin/env bash cd expert MIX_ENV=prod mix release plain --no-compile From a6730263e9bddfa6072a23548bf4d1345548ed62 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 12:53:13 -0400 Subject: [PATCH 11/18] spike: remove generated tests --- engine/test/engine_test.exs | 4 ---- expert/test/expert_test.exs | 4 ---- 2 files changed, 8 deletions(-) diff --git a/engine/test/engine_test.exs b/engine/test/engine_test.exs index 1cb08fe3..77c21a2a 100644 --- a/engine/test/engine_test.exs +++ b/engine/test/engine_test.exs @@ -1,8 +1,4 @@ defmodule EngineTest do use ExUnit.Case doctest Engine - - test "greets the world" do - assert Engine.hello() == :world - end end diff --git a/expert/test/expert_test.exs b/expert/test/expert_test.exs index 2add7537..8c41f2fa 100644 --- a/expert/test/expert_test.exs +++ b/expert/test/expert_test.exs @@ -1,8 +1,4 @@ defmodule ExpertTest do use ExUnit.Case doctest Expert - - test "greets the world" do - assert Expert.hello() == :world - end end From bb53b788904719a5611c7bac3f08925d15ee2e68 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 12:53:36 -0400 Subject: [PATCH 12/18] spike(just): further improve just file --- justfile | 60 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/justfile b/justfile index 729bfdeb..11b87231 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,19 @@ mix_env := env('MIX_ENV', 'dev') build_dir := "_build" / mix_env safe_dir := "_build" / mix_env + "safe" +os := if os() == "macos" { "darwin" } else { os() } +arch := if arch() =~ "(arm|aarch64)" { + "arm64" +} else if arch() =~ "(x86|x86_64)" { + "amd64" +} else { + "unsupported" +} +local_target := if os =~ "(darwin|linux|windows)" { + os + "_" + arch +} else { + "unsupported" +} apps := "expert engine namespace" @@ -15,7 +28,7 @@ run project +ARGS: cd {{project}} eval "{{ ARGS }}" -compile project: +compile project: (deps project) #!/usr/bin/env bash set -euo pipefail @@ -56,41 +69,32 @@ start *opts="--port 9000": build-engine build-expert -e "Application.ensure_all_started(:expert)" \ -- {{opts}} -test: +mix cmd *project: #!/usr/bin/env bash - for project in {{ apps }}; do - ( - cd "$project" + if [ -n "{{ project }}" ]; then + cd {{project}} + mix {{cmd}} + else + for project in {{ apps }}; do + ( + cd "$project" - mix test - ) - done - -format project: - #!/usr/bin/env bash - cd {{project}} - mix format + mix {{ cmd }} + ) + done + fi [unix] release-local: build-engine build-expert #!/usr/bin/env bash cd expert - case "{{os()}}-{{arch()}}" in - "linux-arm" | "linux-aarch64") - target=linux_arm64;; - "linux-x86" | "linux-x86_64") - target=linux_amd64;; - "macos-arm" | "macos-aarch64") - target=darwin_arm64;; - "macos-x86" | "macos-x86_64") - target=darwin_amd64;; - *) - echo "unsupported OS/Arch combination" - exit 1;; - esac - - EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="$target" MIX_ENV=prod mix release --no-compile + + if [ "{{local_target}}" == "unsupported" ]; then + echo "unsupported OS/Arch combination: {{local_target}}" + exit 1 + fi + EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="{{local_target}}" MIX_ENV=prod mix release --no-compile [windows] release-local: build-engine build-expert From 82d6626852bce277140dfe3697ee85004eb230a1 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 12:53:47 -0400 Subject: [PATCH 13/18] spike: add zig and just to .tool-versions --- .tool-versions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.tool-versions b/.tool-versions index e8a5e4e7..ff119615 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,4 @@ elixir 1.17.2-otp-26 erlang 26.2.5 +zig 0.13.0 +just 1.35.0 From 630591b2c94ec658a9c35e0906dcc10a6cfec576 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 12:54:15 -0400 Subject: [PATCH 14/18] spike: remove 7z alias from flake.nix --- flake.nix | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index dfa6185e..e455f9de 100644 --- a/flake.nix +++ b/flake.nix @@ -21,21 +21,13 @@ systems = ["aarch64-darwin" "x86_64-darwin" "x86_64-linux"]; - perSystem = {pkgs, ...}: let - alias_7zz = pkgs.symlinkJoin { - name = "7zz-aliased"; - paths = [pkgs._7zz]; - postBuild = '' - ln -s ${pkgs._7zz}/bin/7zz $out/bin/7z - ''; - }; - in { + perSystem = {pkgs, ...}: { beamWorkspace = { enable = true; devShell = { packages = with pkgs; [ zig - alias_7zz + _7zz just ]; languageServers.elixir = false; From a8d7da92d21669f2c6c7bd78c6c085926e431227 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 13:27:49 -0400 Subject: [PATCH 15/18] spike: fix regression from refactors --- expert/lib/expert/runtime.ex | 2 +- namespace/lib/mix/tasks/namespace.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/expert/lib/expert/runtime.ex b/expert/lib/expert/runtime.ex index 2043c10e..33c8cfa0 100644 --- a/expert/lib/expert/runtime.ex +++ b/expert/lib/expert/runtime.ex @@ -193,7 +193,7 @@ defmodule Expert.Runtime do case connect(node, port, 120) do true -> - {:ok, _} = :rpc.call(node, Application, :ensure_all_started, [:xp_engine]) + {:ok, _} = :rpc.call(node, Application, :ensure_all_started, [:engine]) send(me, {:node, node}) diff --git a/namespace/lib/mix/tasks/namespace.ex b/namespace/lib/mix/tasks/namespace.ex index 9c65e88b..593b845e 100644 --- a/namespace/lib/mix/tasks/namespace.ex +++ b/namespace/lib/mix/tasks/namespace.ex @@ -58,13 +58,13 @@ defmodule Mix.Tasks.Namespace do roots = (roots_from_apps ++ include_roots) -- exclude_roots - opts = [apps: apps, roots: roots, do_apps: options[:apps]] + opts = [apps: apps, roots: roots, do_apps: options[:dot_apps]] Namespace.Transform.Apps.run_all(base_directory, opts) Namespace.Transform.Beams.run_all(base_directory, opts) - if options[:apps] do + if options[:dot_apps] do Namespace.Transform.AppDirectories.run_all(base_directory, opts) end end From b3c02b7ae283ed64675ef780b9646e67fb9a70aa Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 13:28:03 -0400 Subject: [PATCH 16/18] spike(just) improve just file --- justfile | 164 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 83 insertions(+), 81 deletions(-) diff --git a/justfile b/justfile index 11b87231..54824e5a 100644 --- a/justfile +++ b/justfile @@ -1,112 +1,114 @@ mix_env := env('MIX_ENV', 'dev') build_dir := "_build" / mix_env -safe_dir := "_build" / mix_env + "safe" +safe_dir := "_build" / mix_env + "_safe" os := if os() == "macos" { "darwin" } else { os() } -arch := if arch() =~ "(arm|aarch64)" { - "arm64" -} else if arch() =~ "(x86|x86_64)" { - "amd64" -} else { - "unsupported" -} -local_target := if os =~ "(darwin|linux|windows)" { - os + "_" + arch -} else { - "unsupported" -} - +arch := if arch() =~ "(arm|aarch64)" { "arm64" } else { if arch() =~ "(x86|x86_64)" { "amd64" } else { "unsupported" } } +local_target := if os =~ "(darwin|linux|windows)" { os + "_" + arch } else { "unsupported" } apps := "expert engine namespace" +[doc('Run mix deps.get for the given project')] deps project: - #!/usr/bin/env bash - cd {{project}} - mix deps.get + #!/usr/bin/env bash + cd {{ project }} + mix deps.get +[doc('Run an arbitrary command inside the given project directory')] run project +ARGS: - #!/usr/bin/env bash - set -euo pipefail - cd {{project}} - eval "{{ ARGS }}" + #!/usr/bin/env bash + set -euo pipefail + cd {{ project }} + eval "{{ ARGS }}" +[doc('Compile the given project.')] compile project: (deps project) - #!/usr/bin/env bash - set -euo pipefail - - cd {{project}} - # create our safekeeping area - mkdir -p {{safe_dir}} - # delete what is currently in the build dir - rm -rf {{ build_dir }} - # move our build artifacts from safekeeping to the build area - cp -a "{{ safe_dir }}/." "{{ build_dir }}/" - # compile the safe kept code, respects incremental compilation - mix compile - # prep the safe area for new code - rm -rf "{{ safe_dir }}/" - # copy new code in the safe area - cp -a "{{ build_dir }}/." "{{ safe_dir }}/" - + #!/usr/bin/env bash + set -euo pipefail + + cd {{ project }} + # create our safekeeping area + mkdir -p {{ safe_dir }} + # delete what is currently in the build dir + rm -rf {{ build_dir }} + # move our build artifacts from safekeeping to the build area + cp -a "{{ safe_dir }}/." "{{ build_dir }}/" + # compile the safe kept code, respects incremental compilation + mix compile + # prep the safe area for new code + rm -rf "{{ safe_dir }}/" + # copy new code in the safe area + cp -a "{{ build_dir }}/." "{{ safe_dir }}/" + +[private] build project *args: (compile project) - #!/usr/bin/env bash - set -euo pipefail + #!/usr/bin/env bash + set -euo pipefail - cd {{project}} + cd {{ project }} - # namespace the new code - mix namespace --directory "{{ build_dir }}" {{args}} + # namespace the new code + mix namespace --directory "{{ build_dir }}" {{ args }} +[private] build-engine: (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --dot-apps") + +[private] build-expert: (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app req --exclude-app finch --exclude-app nimble_options --exclude-app nimble_pool --exclude-app namespace --exclude-root Jason --include-root Engine") +[doc('Start the local development server')] start *opts="--port 9000": build-engine build-expert - #!/usr/bin/env bash - cd expert - - # no compile is important so it doesn't mess up the namespacing - EXPERT_ENGINE_PATH="../engine/_build/{{ mix_env }}/" mix run \ - --no-compile \ - --no-halt \ - -e "Application.ensure_all_started(:expert)" \ - -- {{opts}} - -mix cmd *project: - #!/usr/bin/env bash - - if [ -n "{{ project }}" ]; then - cd {{project}} - mix {{cmd}} - else - for project in {{ apps }}; do - ( - cd "$project" + #!/usr/bin/env bash + cd expert + + # no compile is important so it doesn't mess up the namespacing + EXPERT_ENGINE_PATH="../engine/_build/{{ mix_env }}/" mix run \ + --no-compile \ + --no-halt \ + -e "Application.ensure_all_started(:expert)" \ + -- {{ opts }} + +[doc('Run a mix command in one or all projects')] +mix cmd *project: + #!/usr/bin/env bash + + if [ -n "{{ project }}" ]; then + cd {{ project }} mix {{ cmd }} - ) - done - fi + else + for project in {{ apps }}; do + ( + cd "$project" + + mix {{ cmd }} + ) + done + fi +[doc('Build a release for the local system')] [unix] release-local: build-engine build-expert - #!/usr/bin/env bash - cd expert + #!/usr/bin/env bash + cd expert - if [ "{{local_target}}" == "unsupported" ]; then - echo "unsupported OS/Arch combination: {{local_target}}" - exit 1 - fi - EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="{{local_target}}" MIX_ENV=prod mix release --no-compile + if [ "{{ local_target }}" == "unsupported" ]; then + echo "unsupported OS/Arch combination: {{ local_target }}" + exit 1 + fi + EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="{{ local_target }}" mix release --no-compile [windows] release-local: build-engine build-expert - # idk actually how to set env vars like this on windows, might crash - EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release --no-compile + # idk actually how to set env vars like this on windows, might crash + EXPERT_RELEASE_MODE=burrito BURRITO_TARGET="windows_amd64" MIX_ENV=prod mix release --no-compile +[doc('Build releases for all target platforms')] release-all: build-engine build-expert - #!/usr/bin/env bash - cd expert - EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release --no-compile + #!/usr/bin/env bash + cd expert + EXPERT_RELEASE_MODE=burrito MIX_ENV=prod mix release --no-compile +[doc('Build a plain release without burrito')] release-plain: build-engine build-expert - #!/usr/bin/env bash - cd expert - MIX_ENV=prod mix release plain --no-compile + #!/usr/bin/env bash + cd expert + MIX_ENV=prod mix release plain --no-compile From 7ba6914e9f2f39c0a42e11602218de437393cd78 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sat, 26 Oct 2024 13:28:09 -0400 Subject: [PATCH 17/18] spike(docs): readme --- README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..fdf6954e --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# Expert + +Welcome to the monorepo for the official Elixir LSP implementation, Expert! + +## Projects + +- `expert` - the LSP server +- `engine` - the code intelligence engine injected into the user's project +- `namespace` - mix task to disguise the engine application to not clobber the user's code + +## Getting Started + +Expert uses the [just](https://just.systems) command runner system (similar to make). If you use [Nix](https://nixos.org/), you can jump in the dev shell `nix develop` and `just` and the rest of the dependencies will be installed for you. + +Otherwise, please install the following dependencies with your choice of package manager or with asdf/mise. + +- [just](https://just.systems) +- Erlang (version found in the .tool-versions file) +- Elixir (version found in the .tool-versions file) +- Zig (version found in the .tool-versions file) +- xz +- 7zz (to create Windows builds) + +To quickly build a release you can run locally + +```shell +# dev build +just release-local + +# prod build +MIX_ENV=prod just release-local +``` +Now a single file executable for your system will be available in `./expert/burrito_out/`, e.g., `./expert/burrito_out/expert_linux_amd64` + +To start the local dev server in TCP mode. + +```shell +just start --port 9000 +``` + +The full set of recipes can be found by running `just --list`. + +``` +Available recipes: + compile project # Compile the given project. + deps project # Run mix deps.get for the given project + mix cmd *project # Run a mix command in one or all projects + release-all # Build releases for all target platforms + release-local # Build a release for the local system + release-plain # Build a plain release without burrito + run project +ARGS # Run an arbitrary command inside the given project directory + start *opts="--port 9000" # Start the local development server +``` From 7035411a01a02abf121b90b3a55ee225c4c68708 Mon Sep 17 00:00:00 2001 From: Mitchell Hanberg Date: Sun, 10 Nov 2024 11:38:53 -0500 Subject: [PATCH 18/18] spike(tests): add integration tests for expert This patch introduces integration tests using GenLSPs builtin testing sdk. The test sdk starts an in process instance of the server and communicates with it via the TCP adapter. In tests, the test is the "client" instead of a text editor. Tests however are not namespaced. From what I could see, tests in Lexical don't seem to be namespaced. Attempting to do so I believe would be a rabbit hole, as you'd have to namespace the in memory compiled tests before they run. --- This also adjusts how the project is namespaced in development. Previously, we would copy the compiled beam files to another directory and namespace the ones in the standard one, then copy the normal files back before compiling again to achieve normal incremental compiles. Now, it leaves the normal files in place, but copies them to another directory and namespaces them there. Then when we start the server, we change the `MIX_BUILD_PATH` to the new directory. --- One last thing to note is that the `:engine` atom is not properly namespaced in the Expert project, as we aren't namespacing applications. But, it was attempting to start the Engine application via `Application.ensure_all_started(:engine)`. To get around this, I put a function in the Engine project that calls that instead, and the Expert projects RPC call is `Engine.ensure_all_started()`. --- README.md | 17 ++-- engine/lib/engine.ex | 3 + expert/lib/expert.ex | 14 ++- expert/lib/expert/release.ex | 2 +- expert/lib/expert/runtime.ex | 5 +- expert/test/expert_test.exs | 169 ++++++++++++++++++++++++++++++++++- expert/test/support/utils.ex | 155 ++++++++++++++++++++++++++++++++ expert/test/test_helper.exs | 4 +- justfile | 72 ++++++++++----- 9 files changed, 404 insertions(+), 37 deletions(-) create mode 100644 expert/test/support/utils.ex diff --git a/README.md b/README.md index fdf6954e..32de5a40 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,13 @@ The full set of recipes can be found by running `just --list`. ``` Available recipes: - compile project # Compile the given project. - deps project # Run mix deps.get for the given project - mix cmd *project # Run a mix command in one or all projects - release-all # Build releases for all target platforms - release-local # Build a release for the local system - release-plain # Build a plain release without burrito - run project +ARGS # Run an arbitrary command inside the given project directory - start *opts="--port 9000" # Start the local development server + compile project # Compile the given project. + deps project # Run mix deps.get for the given project + mix cmd *project # Run a mix command in one or all projects. Use `just test` to run tests. + release-all # Build releases for all target platforms + release-local # Build a release for the local system + release-plain # Build a plain release without burrito + run project +ARGS # Run an arbitrary command inside the given project directory + start *opts="--port 9000" # Start the local development server + test project="all" *args="" # Run tests in the given project ``` diff --git a/engine/lib/engine.ex b/engine/lib/engine.ex index 115929d9..a0a11aed 100644 --- a/engine/lib/engine.ex +++ b/engine/lib/engine.ex @@ -1,2 +1,5 @@ defmodule Engine do + def ensure_all_started() do + Application.ensure_all_started(:engine) + end end diff --git a/expert/lib/expert.ex b/expert/lib/expert.ex index a90d3742..79ab39e6 100644 --- a/expert/lib/expert.ex +++ b/expert/lib/expert.ex @@ -37,7 +37,6 @@ defmodule Expert do }, lsp ) do - System.get_env("EXPERT_ENGINE_PATH") parent = self() name = Path.basename(root_uri) @@ -120,8 +119,16 @@ defmodule Expert do {:reply, symbols, lsp} end - def handle_request(_request, lsp) do - {:noreply, lsp} + def handle_request(%GenLSP.Requests.Shutdown{}, lsp) do + {:reply, nil, assign(lsp, exit_code: 0)} + end + + def handle_request(request, lsp) do + {:reply, + %GenLSP.ErrorResponse{ + code: GenLSP.Enumerations.ErrorCodes.method_not_found(), + message: "Method Not Found: #{request.method}" + }, lsp} end @impl true @@ -138,6 +145,7 @@ defmodule Expert do end def handle_info({:runtime_ready, _name, runtime_pid}, lsp) do + GenLSP.log(lsp, "[Expert] Runtime is ready") Runtime.compile(runtime_pid) {:noreply, assign(lsp, ready: true, runtime: runtime_pid)} diff --git a/expert/lib/expert/release.ex b/expert/lib/expert/release.ex index c5df2853..2d8518dc 100644 --- a/expert/lib/expert/release.ex +++ b/expert/lib/expert/release.ex @@ -2,7 +2,7 @@ defmodule Expert.Release do def assemble(release) do engine_path = Path.expand("../../../engine", __DIR__) - source = Path.join([engine_path, "_build/#{Mix.env()}"]) + source = Path.join([engine_path, "_build/#{Mix.env()}_ns"]) dest = Path.join([ diff --git a/expert/lib/expert/runtime.ex b/expert/lib/expert/runtime.ex index 33c8cfa0..40253cfd 100644 --- a/expert/lib/expert/runtime.ex +++ b/expert/lib/expert/runtime.ex @@ -1,6 +1,7 @@ defmodule Expert.Runtime do @moduledoc false use GenServer + require Logger defguardp is_ready(state) when is_map_key(state, :node) @@ -193,7 +194,9 @@ defmodule Expert.Runtime do case connect(node, port, 120) do true -> - {:ok, _} = :rpc.call(node, Application, :ensure_all_started, [:engine]) + Logger.debug("Going to start the engine") + + {:ok, _} = :rpc.call(node, Engine, :ensure_all_started, []) send(me, {:node, node}) diff --git a/expert/test/expert_test.exs b/expert/test/expert_test.exs index 8c41f2fa..820403fc 100644 --- a/expert/test/expert_test.exs +++ b/expert/test/expert_test.exs @@ -1,4 +1,171 @@ defmodule ExpertTest do use ExUnit.Case - doctest Expert + import GenLSP.Test + import Expert.Support.Utils + + @moduletag :tmp_dir + + @moduletag root_paths: ["my_proj"] + setup %{tmp_dir: tmp_dir} do + File.mkdir_p!(Path.join(tmp_dir, "my_proj/lib")) + File.write!(Path.join(tmp_dir, "my_proj/mix.exs"), mix_exs()) + [cwd: tmp_dir] + end + + setup %{tmp_dir: tmp_dir} do + File.write!(Path.join(tmp_dir, "my_proj/lib/bar.ex"), """ + defmodule Bar do + defstruct [:foo] + + def foo(arg1) do + end + end + """) + + File.write!(Path.join(tmp_dir, "my_proj/lib/code_action.ex"), """ + defmodule Foo.CodeAction do + # some comment + + defmodule NestedMod do + def foo do + :ok + end + end + end + """) + + File.write!(Path.join(tmp_dir, "my_proj/lib/foo.ex"), """ + defmodule Foo do + end + """) + + File.write!(Path.join(tmp_dir, "my_proj/lib/project.ex"), """ + defmodule Project do + def hello do + :world + end + end + """) + + File.rm_rf!(Path.join(tmp_dir, ".elixir-tools")) + + :ok + end + + setup :with_lsp + + test "responds correctly to a shutdown request", %{client: client} do + assert :ok == notify(client, %{method: "initialized", jsonrpc: "2.0", params: %{}}) + + assert :ok == + request(client, %{ + method: "shutdown", + id: 2, + jsonrpc: "2.0" + }) + + assert_result(2, nil) + end + + test "document symbols", %{client: client} = context do + assert :ok == notify(client, %{method: "initialized", jsonrpc: "2.0", params: %{}}) + + assert_notification( + "window/logMessage", + %{ + "message" => "[Expert] Runtime is ready", + "type" => 4 + } + ) + + assert :ok == + request(client, %{ + method: "textDocument/documentSymbol", + id: 2, + jsonrpc: "2.0", + params: %{ + textDocument: %{ + uri: "file://#{Path.join(context.tmp_dir, "my_proj/lib/code_action.ex")}" + } + } + }) + + assert_result(2, [ + %{ + "children" => [ + %{ + "children" => [ + %{ + "children" => [], + "kind" => 12, + "name" => "def foo", + "range" => %{ + "end" => %{"character" => 4, "line" => 6}, + "start" => %{"character" => 4, "line" => 4} + }, + "selectionRange" => %{ + "end" => %{"character" => 4, "line" => 4}, + "start" => %{"character" => 4, "line" => 4} + } + } + ], + "kind" => 2, + "name" => "NestedMod", + "range" => %{ + "end" => %{"character" => 2, "line" => 7}, + "start" => %{"character" => 2, "line" => 3} + }, + "selectionRange" => %{ + "end" => %{"character" => 2, "line" => 3}, + "start" => %{"character" => 2, "line" => 3} + } + } + ], + "kind" => 2, + "name" => "Foo.CodeAction", + "range" => %{ + "end" => %{"character" => 0, "line" => 8}, + "start" => %{"character" => 0, "line" => 0} + }, + "selectionRange" => %{ + "end" => %{"character" => 0, "line" => 0}, + "start" => %{"character" => 0, "line" => 0} + } + } + ]) + end + + test "returns method not found for unimplemented requests", %{client: client} do + id = System.unique_integer([:positive]) + + assert :ok == notify(client, %{method: "initialized", jsonrpc: "2.0", params: %{}}) + + assert :ok == + request(client, %{ + method: "textDocument/signatureHelp", + id: id, + jsonrpc: "2.0", + params: %{position: %{line: 0, character: 0}, textDocument: %{uri: ""}} + }) + + assert_error(^id, %{ + "code" => -32_601, + "message" => "Method Not Found: textDocument/signatureHelp" + }) + end + + test "can initialize the server" do + assert_result(1, %{ + "capabilities" => %{ + "textDocumentSync" => %{ + "openClose" => true, + "save" => %{ + "includeText" => true + }, + "change" => 2 + } + }, + "serverInfo" => %{"name" => "Expert"} + }) + end end diff --git a/expert/test/support/utils.ex b/expert/test/support/utils.ex new file mode 100644 index 00000000..6c03b0bb --- /dev/null +++ b/expert/test/support/utils.ex @@ -0,0 +1,155 @@ +defmodule Expert.Support.Utils do + @moduledoc false + import ExUnit.Assertions + import ExUnit.Callbacks + import GenLSP.Test + + alias GenLSP.Structures.Position + alias GenLSP.Structures.Range + alias GenLSP.Structures.TextEdit + + def mix_exs do + """ + defmodule Project.MixProject do + use Mix.Project + + def project do + [ + app: :project, + version: "0.1.0", + elixir: "~> 1.10", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [] + end + end + """ + end + + def with_lsp(%{tmp_dir: tmp_dir} = context) do + root_paths = + for path <- context[:root_paths] || [""] do + Path.absname(Path.join(tmp_dir, path)) + end + + rvisor = start_supervised!({DynamicSupervisor, [strategy: :one_for_one]}, id: :three) + init_options = context[:init_options] || %{} + + server = server(Expert, dynamic_supervisor: rvisor) + + Process.link(server.lsp) + + client = client(server) + + assert :ok == + request(client, %{ + method: "initialize", + id: 1, + jsonrpc: "2.0", + params: %{ + rootUri: "file://" <> List.first(root_paths), + initializationOptions: init_options, + capabilities: %{ + workspace: %{ + workspaceFolders: false + }, + window: %{ + work_done_progress: false, + showMessage: %{} + } + } + } + }) + + [server: server, client: client] + end + + def uri(path) when is_binary(path) do + URI.to_string(%URI{ + scheme: "file", + host: "", + path: path + }) + end + + defmacro assert_result2( + id, + pattern, + timeout \\ Application.get_env(:ex_unit, :assert_receive_timeout) + ) do + quote do + assert_receive %{ + "jsonrpc" => "2.0", + "id" => unquote(id), + "result" => result + }, + unquote(timeout) + + assert result == unquote(pattern) + end + end + + defmacro did_open(client, file_path, text) do + quote do + assert :ok == + notify(unquote(client), %{ + method: "textDocument/didOpen", + jsonrpc: "2.0", + params: %{ + textDocument: %{ + uri: uri(unquote(file_path)), + text: unquote(text), + languageId: "elixir", + version: 1 + } + } + }) + end + end + + def apply_edit(code, edit) when is_binary(code), do: apply_edit(String.split(code, "\n"), edit) + + def apply_edit(lines, %TextEdit{} = edit) when is_list(lines) do + text = edit.new_text + + %Range{ + start: %Position{line: startl, character: startc}, + end: %Position{line: endl, character: endc} + } = edit.range + + startl_text = Enum.at(lines, startl) + prefix = String.slice(startl_text, 0, startc) + + endl_text = Enum.at(lines, endl) + suffix = String.slice(endl_text, endc, String.length(endl_text) - endc) + + replacement = prefix <> text <> suffix + + new_lines = + Enum.slice(lines, 0, startl) ++ + [replacement] ++ Enum.slice(lines, endl + 1, Enum.count(lines)) + + new_lines + |> Enum.join("\n") + |> String.trim() + end + + defmacro assert_is_text_edit(code, edit, expected) do + quote do + actual = unquote(__MODULE__).apply_edit(unquote(code), unquote(edit)) + assert actual == unquote(expected) + end + end +end diff --git a/expert/test/test_helper.exs b/expert/test/test_helper.exs index 869559e7..cfefcd7e 100644 --- a/expert/test/test_helper.exs +++ b/expert/test/test_helper.exs @@ -1 +1,3 @@ -ExUnit.start() +Logger.configure(level: :warning) + +ExUnit.start(assert_receive_timeout: 30_000) diff --git a/justfile b/justfile index 54824e5a..4b74efd6 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,6 @@ mix_env := env('MIX_ENV', 'dev') build_dir := "_build" / mix_env -safe_dir := "_build" / mix_env + "_safe" +namespaced_dir := "_build" / mix_env + "_ns" os := if os() == "macos" { "darwin" } else { os() } arch := if arch() =~ "(arm|aarch64)" { "arm64" } else { if arch() =~ "(x86|x86_64)" { "amd64" } else { "unsupported" } } local_target := if os =~ "(darwin|linux|windows)" { os + "_" + arch } else { "unsupported" } @@ -21,22 +21,7 @@ run project +ARGS: [doc('Compile the given project.')] compile project: (deps project) - #!/usr/bin/env bash - set -euo pipefail - - cd {{ project }} - # create our safekeeping area - mkdir -p {{ safe_dir }} - # delete what is currently in the build dir - rm -rf {{ build_dir }} - # move our build artifacts from safekeeping to the build area - cp -a "{{ safe_dir }}/." "{{ build_dir }}/" - # compile the safe kept code, respects incremental compilation - mix compile - # prep the safe area for new code - rm -rf "{{ safe_dir }}/" - # copy new code in the safe area - cp -a "{{ build_dir }}/." "{{ safe_dir }}/" + cd {{ project }} && mix compile [private] build project *args: (compile project) @@ -45,29 +30,72 @@ build project *args: (compile project) cd {{ project }} + # remove the existing namespaced dir + rm -rf {{ namespaced_dir }} + # create our namespaced area + mkdir -p {{ namespaced_dir }} + # move our build artifacts from safekeeping to the build area + cp -a "{{ build_dir }}/." "{{ namespaced_dir }}/" + # namespace the new code - mix namespace --directory "{{ build_dir }}" {{ args }} + mix namespace --directory "{{ namespaced_dir }}" {{ args }} [private] -build-engine: (build "engine" "--include-app engine --include-root Engine --exclude-app namespace --dot-apps") +build-engine: (build "engine" "--include-app engine --include-root Engine --dot-apps") [private] -build-expert: (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app req --exclude-app finch --exclude-app nimble_options --exclude-app nimble_pool --exclude-app namespace --exclude-root Jason --include-root Engine") +build-expert: (build "expert" "--include-app expert --exclude-root Expert --exclude-app burrito --exclude-app req --exclude-app finch --exclude-app nimble_options --exclude-app nimble_pool --exclude-app namespace --exclude-root Jason --include-root Engine --include-app engine") + +[doc('Run tests in the given project')] +test project="all" *args="": + MIX_ENV=test just _test {{ project }} {{ args }} + +[private] +_test project="all" *args="": + #!/usr/bin/env bash + set -euo pipefail + + case "{{ project }}" in + expert) + cd {{ project }} + # compile in dev env to simulate normal conditions + # note that we aren't namespacing during the tests + # lexical doesn't seem to namespace as far as I can tell, and + # figuring out how to namespace the test files seemed like a rabbit hole + MIX_ENV=dev just compile engine + export EXPERT_ENGINE_PATH="../engine/_build/dev" + mix compile + mix test --no-compile {{ args }} + ;; + all) + for project in {{ apps }}; do + echo "Testing $project" + just _test "$project" + done + ;; + *) + cd {{ project }} + mix compile + mix test --no-compile {{ args }} + ;; + esac [doc('Start the local development server')] start *opts="--port 9000": build-engine build-expert #!/usr/bin/env bash + set -euo pipefail cd expert # no compile is important so it doesn't mess up the namespacing - EXPERT_ENGINE_PATH="../engine/_build/{{ mix_env }}/" mix run \ + # we set the MIX_BUILD_PATH because we put the namespaced code into a separate directory + MIX_BUILD_PATH="{{ namespaced_dir }}" EXPERT_ENGINE_PATH="{{ "../engine" / namespaced_dir }}" mix run \ --no-compile \ --no-halt \ -e "Application.ensure_all_started(:expert)" \ -- {{ opts }} -[doc('Run a mix command in one or all projects')] +[doc('Run a mix command in one or all projects. Use `just test` to run tests.')] mix cmd *project: #!/usr/bin/env bash