diff --git a/config/dev.exs b/config/dev.exs
index 4a906a5..3d7ed8f 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -1,3 +1,5 @@
import Config
config(:onigumo, :http_client, HTTPoison)
+config(:onigumo, :downloader, Onigumo.Downloader)
+config(:onigumo, :parser, Onigumo.Parser)
diff --git a/config/test.exs b/config/test.exs
index acade56..995249c 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -1,3 +1,4 @@
import Config
config(:onigumo, :http_client, HTTPoisonMock)
+config(:onigumo, :downloader, OnigumoDownloaderMock)
diff --git a/lib/cli.ex b/lib/cli.ex
index 0a0f638..4ace1e4 100644
--- a/lib/cli.ex
+++ b/lib/cli.ex
@@ -1,7 +1,37 @@
defmodule Onigumo.CLI do
- def main([component]) do
- module = Module.safe_concat("Onigumo", component)
- root_path = File.cwd!()
- module.main(root_path)
+ @components %{
+ :downloader => Application.compile_env(:onigumo, :downloader),
+ :parser => Application.compile_env(:onigumo, :parser)
+ }
+
+ def main(argv) do
+ case OptionParser.parse(
+ argv,
+ aliases: [C: :working_dir],
+ strict: [working_dir: :string]
+ ) do
+ {parsed_switches, [component], []} ->
+ {:ok, module} = Map.fetch(@components, String.to_atom(component))
+ working_dir = Keyword.get(parsed_switches, :working_dir, File.cwd!())
+ module.main(working_dir)
+
+ _ ->
+ usage_message()
+ end
+ end
+
+ defp usage_message() do
+ components = Enum.join(Map.keys(@components), ", ")
+
+ IO.puts("""
+ Usage: onigumo [OPTION]... [COMPONENT]
+
+ Simple program that retrieves HTTP web content as structured data.
+
+ COMPONENT\tOnigumo component to run, available: #{components}
+
+ OPTIONS:
+ -C, --working-dir
\tChange working dir to before running
+ """)
end
end
diff --git a/lib/onigumo/component.ex b/lib/onigumo/component.ex
new file mode 100644
index 0000000..f706af2
--- /dev/null
+++ b/lib/onigumo/component.ex
@@ -0,0 +1,4 @@
+defmodule Onigumo.Component do
+ @doc "Runs the component."
+ @callback main(root_path :: String.t()) :: :ok
+end
diff --git a/lib/onigumo/downloader.ex b/lib/onigumo/downloader.ex
index fa0606b..bf9ddf7 100644
--- a/lib/onigumo/downloader.ex
+++ b/lib/onigumo/downloader.ex
@@ -2,7 +2,9 @@ defmodule Onigumo.Downloader do
@moduledoc """
Web scraper
"""
+ @behaviour Onigumo.Component
+ @impl Onigumo.Component
def main(root_path) do
http_client().start()
diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex
index 7942e57..1939ad5 100644
--- a/lib/onigumo/parser.ex
+++ b/lib/onigumo/parser.ex
@@ -2,11 +2,15 @@ defmodule Onigumo.Parser do
@moduledoc """
Web scraper
"""
+ @behaviour Onigumo.Component
+ @impl Onigumo.Component
def main(root_path) do
root_path
|> list_downloaded()
|> Enum.map(&IO.puts(&1))
+
+ :ok
end
defp list_downloaded(path) do
@@ -17,6 +21,6 @@ defmodule Onigumo.Parser do
defp is_downloaded(path) do
suffix = Application.get_env(:onigumo, :downloaded_suffix)
- Path.extname(path) == ".#{suffix}"
+ Path.extname(path) == suffix
end
end
diff --git a/mix.exs b/mix.exs
index 77e7906..2e64791 100644
--- a/mix.exs
+++ b/mix.exs
@@ -2,13 +2,16 @@ defmodule Onigumo.MixProject do
use Mix.Project
def project do
+ env = Mix.env()
+
[
app: :onigumo,
version: "0.1.0",
elixir: "~> 1.10",
- start_permanent: Mix.env() == :prod,
+ start_permanent: env == :prod,
deps: deps(),
- escript: escript()
+ escript: escript(),
+ elixirc_paths: elixirc_paths(env)
]
end
@@ -37,4 +40,10 @@ defmodule Onigumo.MixProject do
main_module: Onigumo.CLI
]
end
+
+ defp elixirc_paths(:test), do: elixirc_paths_default() ++ ["test/support"]
+
+ defp elixirc_paths(_), do: elixirc_paths_default()
+
+ defp elixirc_paths_default(), do: Mix.Project.config()[:elixirc_paths]
end
diff --git a/mix.lock b/mix.lock
index 628ea6a..53afdb2 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,14 +1,14 @@
%{
- "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
- "floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"},
- "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"},
+ "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
+ "floki": {:hex, :floki, "0.35.4", "cc947b446024732c07274ac656600c5c4dc014caa1f8fb2dfff93d275b83890d", [:mix], [], "hexpm", "27fa185d3469bd8fc5947ef0f8d5c4e47f0af02eb6b070b63c868f69e3af0204"},
+ "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~>2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
"html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
"httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
"idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
"mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
- "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"},
- "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
- "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
+ "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
+ "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
+ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
}
diff --git a/test/onigumo_cli_test.exs b/test/onigumo_cli_test.exs
new file mode 100644
index 0000000..c0d51ba
--- /dev/null
+++ b/test/onigumo_cli_test.exs
@@ -0,0 +1,68 @@
+defmodule OnigumoCLITest do
+ use ExUnit.Case
+ import ExUnit.CaptureIO
+ import Mox
+
+ @invalid_arguments [
+ "Downloader",
+ "uploader"
+ ]
+
+ @invalid_switches [
+ "--invalid",
+ "-c"
+ ]
+
+ @working_dir_switches [
+ "--working-dir",
+ "-C"
+ ]
+
+ describe("Onigumo.CLI.main/1") do
+ for argument <- @invalid_arguments do
+ test("run CLI with invalid argument #{inspect(argument)}") do
+ assert_raise(MatchError, fn -> Onigumo.CLI.main([unquote(argument)]) end)
+ end
+ end
+
+ test("run CLI with no arguments") do
+ assert usage_message_printed?(fn -> Onigumo.CLI.main([]) end)
+ end
+
+ test("run CLI with more than one argument") do
+ assert usage_message_printed?(fn -> Onigumo.CLI.main(["Downloader", "Parser"]) end)
+ end
+
+ for switch <- @invalid_switches do
+ test("run CLI with invalid switch #{inspect(switch)}") do
+ assert usage_message_printed?(fn -> Onigumo.CLI.main([unquote(switch)]) end)
+ end
+ end
+
+ @tag :tmp_dir
+ test("run CLI with 'downloader' argument passing cwd", %{tmp_dir: tmp_dir}) do
+ expect(OnigumoDownloaderMock, :main, fn working_dir -> working_dir end)
+
+ File.cd(tmp_dir)
+ assert Onigumo.CLI.main(["downloader"]) == tmp_dir
+ end
+
+ for switch <- @working_dir_switches do
+ @tag :tmp_dir
+ test("run CLI 'downloader' with #{inspect(switch)} switch", %{tmp_dir: tmp_dir}) do
+ expect(OnigumoDownloaderMock, :main, fn working_dir -> working_dir end)
+
+ assert Onigumo.CLI.main(["downloader", unquote(switch), tmp_dir]) == tmp_dir
+ end
+
+ test("run CLI 'downloader' with #{inspect(switch)} without any value") do
+ assert usage_message_printed?(fn -> Onigumo.CLI.main(["downloader", unquote(switch)]) end)
+ end
+ end
+
+ defp usage_message_printed?(function) do
+ output = capture_io(function)
+ String.starts_with?(output, "Usage: onigumo ")
+ end
+ end
+end
diff --git a/test/onigumo_downloader_test.exs b/test/onigumo_downloader_test.exs
index b40bc8c..7e891d8 100644
--- a/test/onigumo_downloader_test.exs
+++ b/test/onigumo_downloader_test.exs
@@ -14,11 +14,11 @@ defmodule OnigumoDownloaderTest do
@tag :tmp_dir
test("run Downloader", %{tmp_dir: tmp_dir}) do
expect(HTTPoisonMock, :start, fn -> nil end)
- expect(HTTPoisonMock, :get!, length(@urls), &prepare_response/1)
+ expect(HTTPoisonMock, :get!, length(@urls), &HttpSupport.response/1)
input_path_env = Application.get_env(:onigumo, :input_path)
input_path_tmp = Path.join(tmp_dir, input_path_env)
- input_file_content = prepare_input(@urls)
+ input_file_content = InputSupport.url_list(@urls)
File.write!(input_path_tmp, input_file_content)
Onigumo.Downloader.main(tmp_dir)
@@ -30,11 +30,11 @@ defmodule OnigumoDownloaderTest do
describe("Onigumo.Downloader.create_download_stream/1") do
@tag :tmp_dir
test("download URLs from the input file with a created stream", %{tmp_dir: tmp_dir}) do
- expect(HTTPoisonMock, :get!, length(@urls), &prepare_response/1)
+ expect(HTTPoisonMock, :get!, length(@urls), &HttpSupport.response/1)
input_path_env = Application.get_env(:onigumo, :input_path)
input_path_tmp = Path.join(tmp_dir, input_path_env)
- input_file_content = prepare_input(@urls)
+ input_file_content = InputSupport.url_list(@urls)
File.write!(input_path_tmp, input_file_content)
Onigumo.Downloader.create_download_stream(tmp_dir) |> Stream.run()
@@ -46,7 +46,7 @@ defmodule OnigumoDownloaderTest do
describe("Onigumo.Downloader.download_url/2") do
@tag :tmp_dir
test("download a URL", %{tmp_dir: tmp_dir}) do
- expect(HTTPoisonMock, :get!, &prepare_response/1)
+ expect(HTTPoisonMock, :get!, &HttpSupport.response/1)
input_url = Enum.at(@urls, 0)
Onigumo.Downloader.download_url(input_url, tmp_dir)
@@ -54,18 +54,18 @@ defmodule OnigumoDownloaderTest do
output_file_name = Onigumo.Downloader.create_file_name(input_url)
output_path = Path.join(tmp_dir, output_file_name)
read_output = File.read!(output_path)
- expected_output = body(input_url)
+ expected_output = HttpSupport.body(input_url)
assert(read_output == expected_output)
end
end
describe("Onigumo.Downloader.get_url/1") do
test("get response by HTTP request") do
- expect(HTTPoisonMock, :get!, &prepare_response/1)
+ expect(HTTPoisonMock, :get!, &HttpSupport.response/1)
url = Enum.at(@urls, 0)
get_response = Onigumo.Downloader.get_url(url)
- expected_response = prepare_response(url)
+ expected_response = HttpSupport.response(url)
assert(get_response == expected_response)
end
end
@@ -73,9 +73,9 @@ defmodule OnigumoDownloaderTest do
describe("Onigumo.Downloader.get_body/1") do
test("extract body from URL response") do
url = Enum.at(@urls, 0)
- response = prepare_response(url)
+ response = HttpSupport.response(url)
get_body = Onigumo.Downloader.get_body(response)
- expected_body = body(url)
+ expected_body = HttpSupport.body(url)
assert(get_body == expected_body)
end
end
@@ -101,7 +101,7 @@ defmodule OnigumoDownloaderTest do
input_path_env = Application.get_env(:onigumo, :input_path)
input_path_tmp = Path.join(tmp_dir, input_path_env)
- input_file_content = prepare_input(input_urls)
+ input_file_content = InputSupport.url_list(input_urls)
File.write!(input_path_tmp, input_file_content)
loaded_urls = Onigumo.Downloader.load_urls(tmp_dir) |> Enum.to_list()
@@ -124,27 +124,11 @@ defmodule OnigumoDownloaderTest do
end
end
- defp prepare_response(url) do
- %HTTPoison.Response{
- status_code: 200,
- body: body(url)
- }
- end
-
- defp prepare_input(urls) do
- Enum.map(urls, &(&1 <> "\n"))
- |> Enum.join()
- end
-
- defp body(url) do
- "Body from: #{url}\n"
- end
-
defp assert_downloaded(url, tmp_dir) do
file_name = Onigumo.Downloader.create_file_name(url)
output_path = Path.join(tmp_dir, file_name)
read_output = File.read!(output_path)
- expected_output = body(url)
+ expected_output = HttpSupport.body(url)
assert(read_output == expected_output)
end
end
diff --git a/test/support/http.ex b/test/support/http.ex
new file mode 100644
index 0000000..6a0b61c
--- /dev/null
+++ b/test/support/http.ex
@@ -0,0 +1,12 @@
+defmodule HttpSupport do
+ def response(url) do
+ %HTTPoison.Response{
+ status_code: 200,
+ body: body(url)
+ }
+ end
+
+ def body(url) do
+ "Body from: #{url}\n"
+ end
+end
diff --git a/test/support/input.ex b/test/support/input.ex
new file mode 100644
index 0000000..a73a4fa
--- /dev/null
+++ b/test/support/input.ex
@@ -0,0 +1,6 @@
+defmodule InputSupport do
+ def url_list(urls) do
+ Enum.map(urls, &(&1 <> "\n"))
+ |> Enum.join()
+ end
+end
diff --git a/test/test_helper.exs b/test/test_helper.exs
index e6511e3..e16a0ae 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -1,3 +1,4 @@
ExUnit.start()
Mox.defmock(HTTPoisonMock, for: HTTPoison.Base)
+Mox.defmock(OnigumoDownloaderMock, for: Onigumo.Component)