add function to ETL quotes from various sources. remove duplicates, e…

…liminate null, nil and "unknown" #1

lib/clean.ex

@@ -0,0 +1,21 @@
+defmodule Clean do
+  def remove_noise do
+    {:ok, data} = File.read("quotes_raw.json")
+    list = Jason.decode!(data)
+    IO.inspect Enum.count(list)
+    clean = list
+    |> Enum.uniq
+    |> Enum.map(fn q ->
+      if q["author"] != "" && q["author"] != nil do
+        q
+      end
+    end)
+    |> Enum.filter(& !is_nil(&1))
+    |> Enum.sort_by(fn q ->
+      q["author"]
+    end)
+
+    IO.inspect Enum.count(clean)
+    File.write!("quotes.json", Jason.encode!(clean))
+  end
+end