Clean up and format a little

config/config.exs

@@ -27,4 +27,4 @@ use Mix.Config
 # Configuration from the imported file will override the ones defined
 # here (which is why it is important to import them last).
 #
-import_config "#{Mix.env}.exs"
+import_config "#{Mix.env()}.exs"

lib/gen_spider.ex

@@ -1,15 +1,15 @@
 defmodule GenSpider do
   @moduledoc "README.md"
-              |> File.read!()
-              |> String.split("<!-- MDOC !-->")
-              |> Enum.fetch!(1)
+             |> File.read!()
+             |> String.split("<!-- MDOC !-->")
+             |> Enum.fetch!(1)
 
   require Logger
 
   @typedoc "Options used by the `start*` functions"
   @type options :: [options]
 
-  @type option :: {:name, GenServer.name}
+  @type option :: {:name, GenServer.name()}
 
   @typep state :: any
 
@@ -35,6 +35,8 @@ defmodule GenSpider do
 
   # Define the callbacks for `GenSpider`
   @callback init(any) ::
-    {:ok, state} | {:ok, state, timeout | :hibernate} |
-    :ignore | {:stop, reason :: term}
+              {:ok, state}
+              | {:ok, state, timeout | :hibernate}
+              | :ignore
+              | {:stop, reason :: term}
 end

mix.exs

@@ -1,14 +1,26 @@
 defmodule GenSpider.MixProject do
   use Mix.Project
 
+  @version "0.1.0"
+
   def project do
     [
       app: :gen_spider,
-      version: "0.1.0",
+      version: @version,
       elixir: "~> 1.6",
-      elixirc_paths: elixirc_paths(Mix.env),
+      elixirc_paths: elixirc_paths(Mix.env()),
       start_permanent: Mix.env() == :prod,
-      deps: deps()
+      deps: deps(),
+      package: package(),
+      docs: docs(),
+      aliases: aliases(),
+      name: "GenSpider",
+      source_url: "https://github.com/sntran/gen_spider",
+      homepage_url: "http://sntran.github.io/gen_spider",
+      description: """
+      A behaviour for defining Spiders that crawl and parse pages
+      for a particular site (or, in some cases, a group of sites).
+      """
     ]
   end
 
@@ -22,15 +34,42 @@ defmodule GenSpider.MixProject do
 
   # Specifies which paths to compile per environment.
   defp elixirc_paths(:test), do: ["lib", "test/support"]
-  defp elixirc_paths(_),     do: ["lib"]
+  defp elixirc_paths(_), do: ["lib"]
 
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
       # {:dep_from_hexpm, "~> 0.3.0"},
       # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
       {:hackney, "~> 1.13"},
+
+      # Test dependencies
       {:mix_test_watch, "~> 0.8", only: :dev, runtime: false}
     ]
   end
+
+  defp package do
+    [
+      maintainers: [
+        "Son Tran-Nguyen"
+      ],
+      licenses: ["Apache 2.0"],
+      links: %{github: "https://github.com/sntran/gen_spider"},
+      files: ~w(assets lib priv) ++ ~w(CHANGELOG.md LICENSE mix.exs README.md)
+    ]
+  end
+
+  defp docs do
+    [
+      main: "GenSpider",
+      source_ref: "v#{@version}"
+    ]
+  end
+
+  defp aliases do
+    [
+      format: ["format --check-formatted --check-equivalent"],
+      test: ["test --cover"]
+    ]
+  end
 end

mix.lock

@@ -1,8 +1,6 @@
 %{
   "certifi": {:hex, :certifi, "2.3.1", "d0f424232390bf47d82da8478022301c561cf6445b5b5fb6a84d49a9e76d2639", [:rebar3], [{:parse_trans, "3.2.0", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
-  "cortex": {:git, "https://github.com/urbint/cortex.git", "38ca1262293a76f67d139e7682d836afc6294512", []},
-  "file_system": {:hex, :file_system, "0.2.4", "f0bdda195c0e46e987333e986452ec523aed21d784189144f647c43eaf307064", [:mix], [], "hexpm"},
-  "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"},
+  "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"},
   "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
   "idna": {:hex, :idna, "5.1.2", "e21cb58a09f0228a9e0b95eaa1217f1bcfc31a1aaa6e1fdf2f53a33f7dbd9494", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},

src/gen_spider.erl

@@ -24,7 +24,7 @@
 ]).
 
 -include("gen_spider.hrl").
--include_lib("gen_spider_internal.hrl").
+-include("gen_spider_internal.hrl").
 
 -record(spider, {
   module :: module(),

test/gen_spider_test.exs

@@ -4,44 +4,39 @@ defmodule GenSpiderTest do
 
   describe "@callback init/1" do
     test "is called when started" do
-      TestSpider.start([
-        init: fn(_args) ->
+      TestSpider.start(
+        init: fn _args ->
           Kernel.send(:tester, {:called_back, :init, 1})
         end
-      ])
+      )
+
       assert_received {:called_back, :init, 1}, "@callback init/1 was not called"
     end
 
     test "should have args passed through" do
       callbacks = [
-        init: fn(args) ->
+        args: 1,
+        init: fn args ->
           Kernel.send(:tester, {:called_back, :init, args})
         end
       ]
+
       TestSpider.start(callbacks)
-      assert_received {:called_back, :init, callbacks}, "@callback init/1 did not receive args"
+      assert_received {:called_back, :init, 1}, "@callback init/1 did not receive args"
     end
 
     test "must return initial state" do
-      assert {:error, _reason} = TestSpider.start([
-        init: fn(_args) -> :ok end
-      ])
-      assert {:ok, pid} = TestSpider.start([
-        init: fn(_args) -> {:ok, 1} end
-      ])
+      assert {:error, _reason} = TestSpider.start(init: fn _args -> :ok end)
+      assert {:ok, _pid} = TestSpider.start(init: fn _args -> {:ok, 1} end)
     end
 
     test "can ignore initalization" do
-      assert :ignore = TestSpider.start([
-        init: fn(_args) -> :ignore end
-      ])
+      assert :ignore = TestSpider.start(init: fn _args -> :ignore end)
     end
 
     test "can stop with error" do
       reason = "test"
-      assert {:error, ^reason} = TestSpider.start([
-        init: fn(_args) -> {:stop, reason} end
-      ])
+      assert {:error, ^reason} = TestSpider.start(init: fn _args -> {:stop, reason} end)
     end
   end
 
@@ -53,51 +48,73 @@ defmodule GenSpiderTest do
     end
 
     test "is called when URLs are not provided in option for `start/3`" do
-      TestSpider.start([
-        start_requests: fn(_state) ->
+      TestSpider.start(
+        start_requests: fn _state ->
           Kernel.send(:tester, {:called_back, :start_requests, 1})
         end
-      ])
+      )
 
-      assert_received {:called_back, :start_requests, 1}, "@callback start_requests/1 was not called"
+      assert_received {:called_back, :start_requests, 1},
+                      "@callback start_requests/1 was not called"
     end
 
     test "is not called when URLs are provided in option for `start/3`" do
-      TestSpider.start([
-        start_requests: fn(_state) ->
-          Kernel.send(:tester, {:called_back, :start_requests, 1})
-        end
-      ], [start_urls: ["http://www.example.com"]])
+      TestSpider.start(
+        [
+          start_requests: fn _state ->
+            Kernel.send(:tester, {:called_back, :start_requests, 1})
+          end
+        ],
+        start_urls: ["http://www.example.com"]
+      )
 
-      refute_receive {:called_back, :start_requests, 1}, 100, "@callback start_requests/1 was not supposed to be called"
+      refute_receive {:called_back, :start_requests, 1},
+                     100,
+                     "@callback start_requests/1 was not supposed to be called"
     end
 
     test "will be delayed if specified in the return of @callback init/1" do
-      TestSpider.start([
-        init: fn(args) ->
+      TestSpider.start(
+        init: fn args ->
           {:ok, args, 100}
         end,
-        start_requests: fn(_state) ->
+        start_requests: fn _state ->
           Kernel.send(:tester, {:called_back, :start_requests, 1})
         end
-      ])
+      )
+
+      refute_receive {:called_back, :start_requests, 1},
+                     100,
+                     "@callback start_requests/1 was not supposed to be called before the delay"
 
-      refute_receive {:called_back, :start_requests, 1}, 100, "@callback start_requests/1 was not supposed to be called before the delay"
-      assert_receive {:called_back, :start_requests, 1}, 101, "@callback start_requests/1 was not called after delay"
+      assert_receive {:called_back, :start_requests, 1},
+                     101,
+                     "@callback start_requests/1 was not called after delay"
     end
 
     test "must return list of Requests" do
-      {:ok, pid1} = TestSpider.start_link([
-        start_requests: fn(_state) -> :ok end
-      ])
+      {:ok, pid1} = TestSpider.start_link(start_requests: fn _state -> :ok end)
       assert_receive {:EXIT, ^pid1, :invalid_return}
 
-      {:ok, pid2} = TestSpider.start([
-        start_requests: fn(state) ->
-          {:ok, [], state}
-        end
-      ])
+      {:ok, pid2} =
+        TestSpider.start(
+          start_requests: fn state ->
+            {:ok, [], state}
+          end
+        )
+
       refute_receive {:EXIT, ^pid2, :invalid_return}
     end
+
+    test "is optional even when there is no `start_urls`" do
+      {:ok, pid} =
+        TestSpider.start(
+          init: fn args ->
+            {:ok, args, 100}
+          end
+        )
+
+      refute_receive {:EXIT, ^pid, :invalid_return}
+    end
   end
 end

test/support/test_case.ex

@@ -12,35 +12,8 @@ defmodule GenSpider.TestCase do
     Process.register(self(), :tester)
 
     unless tags[:async] do
-
     end
 
     :ok
   end
-
-  # Quick macro to generate a spider module that implements :gen_spider
-  defmacro gen_spider(module, callbacks) do
-    unless Keyword.keyword?(callbacks) do
-      raise ArgumentError, "second argument to `spider` must be a compile time keyword list"
-    end
-
-    callbacks = Macro.escape(callbacks)
-
-    quote bind_quoted: [module: module, callbacks: callbacks] do
-      defmodule module do
-        @behaviour :gen_spider
-
-        callbacks = Keyword.put_new(callbacks, :init, quote do
-          fn(args) -> {:ok, args} end
-        end)
-
-        for {name, {:fn, _, [{:->, _, [args, body]}]}} <- callbacks do
-          @impl true
-          def unquote(name)(unquote_splicing(args)) do
-            unquote(body)
-          end
-        end
-      end
-    end
-  end
 end

test/support/test_spider.ex

@@ -8,7 +8,6 @@ defmodule TestSpider do
   To start a test spider, call `start_link/2` with a keyword list
   of callbacks to test, in the form of {name, function}.
   """
-  alias :gen_spider, as: GenSpider
 
   @behaviour GenSpider
 
@@ -17,32 +16,36 @@ defmodule TestSpider do
   @typep spider :: list()
 
   ## API
-  @spec start([callback], [GenSpider.option]) :: :ignore | {:error, any()} | {:ok, pid()}
+  @spec start([callback], [GenSpider.option()]) :: :ignore | {:error, any()} | {:ok, pid()}
   def start(callbacks, options \\ []) do
     GenSpider.start(__MODULE__, callbacks, options)
   end
 
-  @spec start_link([callback], [GenSpider.option]) :: :ignore | {:error, any()} | {:ok, pid()}
+  @spec start_link([callback], [GenSpider.option()]) :: :ignore | {:error, any()} | {:ok, pid()}
   def start_link(callbacks, options \\ []) do
     GenSpider.start_link(__MODULE__, callbacks, options)
   end
 
   ## GenSpider callbacks
 
   @impl true
-  @spec init(keyword()) :: {:ok, state}
-                          | :ignore
-                          | {:stop, reason :: term}
+  @spec init(keyword()) ::
+          {:ok, state}
+          | :ignore
+          | {:stop, reason :: term}
   def init(options \\ []) do
     # Init arguments if any.
     args = Keyword.get(options, :args)
+
     case maybe_apply(options, :init, [args], {:ok, args}) do
       {:ok, state} ->
         spider = Keyword.put(options, :state, state)
         {:ok, spider}
+
       {:ok, state, delay} ->
         spider = Keyword.put(options, :state, state)
         {:ok, spider, delay}
+
       other ->
         other
     end
@@ -52,9 +55,11 @@ defmodule TestSpider do
   @spec start_requests(spider()) :: {:ok, list(), state()}
   def start_requests(spider) do
     state = spider[:state]
+
     case maybe_apply(spider, :start_requests, [state], {:ok, [], state}) do
       {:ok, fetches, state} ->
         {:ok, fetches, Keyword.put(spider, :state, state)}
+
       other ->
         other
     end
@@ -64,6 +69,7 @@ defmodule TestSpider do
     case Keyword.get(spider, fun) do
       nil ->
         default_reply
+
       callback when is_function(callback) ->
         apply(callback, args)
     end