followers spider

lib/octo/client.ex

@@ -24,6 +24,16 @@ defmodule Octograph.Octo.Client do
 
 	def client, do:	%Tentacat.Client{auth: %{access_token: access_token}}
 
+	def all_followers(user) do
+		pages = div(user.flrs_count, 100) + 1
+		(1..pages)
+		|> Enum.map(fn(page) ->
+			Tentacat.Users.Followers.followers(user.login, client, page)
+		end)
+	  |> List.flatten
+	end
+
+
 	defp access_token, do: Application.get_env(:octograph, :github_token)
 
 

lib/octo/user_spider.ex

@@ -18,6 +18,7 @@
 	end
 
 	def info do
+		IO.inspect "1111111111"
 		if Process.whereis(Octograph.Octo.UserSpider) do
 			state = GenServer.call(__MODULE__, :info, :infinity)
 			%{status: :active, state: state}
@@ -30,7 +31,7 @@
 		users = Octograph.Octo.Client.users(since)
 		Enum.each users, fn(user) ->
 			user_node = Octograph.UserNode.new(user)
-			Octograph.UserNodeRepo.update_or_create(user_node)
+			Octograph.UserNodeRepo.find_or_create(user_node)
 		end
 		last = List.last(users)
 		:timer.sleep(500)

lib/traverser.ex

@@ -2,8 +2,9 @@ defmodule Octograph.Traverser do
 
 
 	def level(n) do
-		acc0 = {[Octograph.UserNodeRepo.find_by_login("antonmi")], []}
-		# acc0 = {[Octograph.UserNodeRepo.sample], []}
+		# acc0 = {[Octograph.UserNodeRepo.find_by_login("antonmi")], []}
+		# acc0 = {[Octograph.UserNodeRepo.with_max_followers], []}
+		acc0 = {[Octograph.UserNodeRepo.sample], []}
 		Enum.reduce((1..n), acc0, fn(_i, acc) -> 
 			{nodes, edges} = acc
 			new_edges = Octograph.FollowEdgeRepo.from_nodes(nodes)

mix.lock

@@ -2,7 +2,7 @@
   "cowboy": {:hex, :cowboy, "1.0.0"},
   "cowlib": {:hex, :cowlib, "1.0.1"},
   "decimal": {:hex, :decimal, "1.1.0"},
-  "ecto": {:hex, :ecto, "0.10.1"},
+  "ecto": {:hex, :ecto, "0.10.2"},
   "espec": {:hex, :espec, "0.4.1"},
   "exjsx": {:hex, :exjsx, "3.1.0"},
   "fs": {:hex, :fs, "0.9.1"},
@@ -13,12 +13,12 @@
   "meck": {:hex, :meck, "0.8.2"},
   "mongo": {:hex, :mongo, "0.5.0"},
   "phoenix": {:hex, :phoenix, "0.11.0"},
-  "phoenix_ecto": {:hex, :phoenix_ecto, "0.3.0"},
+  "phoenix_ecto": {:hex, :phoenix_ecto, "0.3.1"},
   "phoenix_live_reload": {:hex, :phoenix_live_reload, "0.3.0"},
   "plug": {:hex, :plug, "0.11.3"},
-  "poison": {:hex, :poison, "1.3.1"},
+  "poison": {:hex, :poison, "1.4.0"},
   "poolboy": {:hex, :poolboy, "1.4.2"},
   "postgrex": {:hex, :postgrex, "0.8.1"},
   "ranch": {:hex, :ranch, "1.0.0"},
   "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.4"},
-  "tentacat": {:git, "git://github.com/antonmi/tentacat.git", "37a5db32df1bf4151cec83f2e023ea2cbd89f286", []}}
+  "tentacat": {:git, "git://github.com/antonmi/tentacat.git", "709bce4a28d06a4b21460c230dd571692db3a9e7", []}}

spec/octo/client_spec.exs

@@ -49,10 +49,23 @@ defmodule Octo.ClientSpec do
 
 	end
 
-	describe "connections" do
-	  it do
-
-	  end
+	describe "all_followers" do
+		before do
+			count = Octograph.Octo.Client.users("brianleroux")["followers"]
+			data = %{"login" => "brianleroux", "id" => 990}
+			user = %{Octograph.UserNode.new(data) | flrs_count: count}
+			{:ok, user: user, count: count}
+		end
+
+		let :all_followers, do: Octograph.Octo.Client.all_followers(__.user)
+
+		it "checks followers" do
+			followers_logins = all_followers 
+			|> Enum.map(fn(f) -> f["login"] end)
+			|> Enum.uniq 
+			Enum.count(followers_logins) |> should eq __.count
+		end
+
 	end
 
 end

spec/repos/follow_edge_repo_spec.exs

@@ -28,6 +28,18 @@ defmodule FollowEdgeRepoSpec do
 	it do: Octograph.UserNodeRepo.count |> should eq 3
 	it do: Octograph.FollowEdgeRepo.count |> should eq 3
 
+	describe "duplicate records" do
+		before do
+		 	edge1 = Octograph.FollowEdge.new(__.u1.id, __.u2.id)
+		 	|> Octograph.FollowEdgeRepo.create
+		 	edge2 = Octograph.FollowEdge.new(__.u1.id, __.u3.id)
+		 	|> Octograph.FollowEdgeRepo.create
+		 	edge3 = Octograph.FollowEdge.new(__.u2.id, __.u1.id)
+		 	|> Octograph.FollowEdgeRepo.create
+	 	end
+	 	it do: Octograph.FollowEdgeRepo.count |> should eq 3
+	end
+
 	describe "edges" do
 		let :from, do: Octograph.FollowEdgeRepo.from_nodes([__.u1])
 		let :to, do: Octograph.FollowEdgeRepo.to_nodes([__.u1])
@@ -38,6 +50,7 @@ defmodule FollowEdgeRepoSpec do
 		end
 	end
 
+
 
 
 end

spec/repos/user_node_repo_spec.exs

@@ -31,15 +31,15 @@ defmodule UserNodeRepoSpec do
 		it do: last.github_id |> should eq 3
 	end
 
-	describe "update_or_create" do
+	describe "find_or_create" do
 		before do
 			data = %{"login" => "test", "id" => 100500}
 			user_node = Octograph.UserNode.new(data)
 			{:ok, user_node: user_node}
 		end
 
 		before do
-			created = Octograph.UserNodeRepo.update_or_create(__.user_node)
+			created = Octograph.UserNodeRepo.find_or_create(__.user_node)
 			{:ok, created: created}
 		end
 
@@ -50,22 +50,6 @@ defmodule UserNodeRepoSpec do
 			expect(user_node.github_id).to eq(100500)
 		end
 
-		context "udpdate" do
-			before do
-				data = %{"login" => "test", "id" => 100501}
-				user_node = Octograph.UserNode.new(data)
-				Octograph.UserNodeRepo.update_or_create(user_node)
-			end
-
-			let :user_node, do: Octograph.UserNodeRepo.find_by_login("test")
-
-			it "check user_node" do
-				expect(user_node.login).to eq("test")
-				expect(user_node.github_id).to eq(100501)
-			end
-		end
-
-
 	end
 
 	describe "followers_updated" do
@@ -81,12 +65,48 @@ defmodule UserNodeRepoSpec do
 	end
 
 
-	describe "without_followers" do
+	describe "first_without_followers" do
 		before do
 			Octograph.UserNode.followers_updated!(__.u1)
 		end
 
-		let :users, do: Octograph.UserNodeRepo.without_followers
-		it do: Enum.map(users, &(&1.login)) |> should eq ["u2", "u3"]
+		let :user, do: Octograph.UserNodeRepo.first_without_followers
+		it do: user.login |> should eq "u2"
+	end
+
+	describe "find_by_logins" do
+		let :users, do: Octograph.UserNodeRepo.find_by_logins(["u1", "u2"])
+
+		it do: Enum.map(users, &(&1.login)) |> should eq(["u1", "u2"])
+	end
+
+	describe "find_or_create_followers" do
+		let :data do
+			[
+				%{"login" => "a1", "id" => 11},
+				%{"login" => "u1", "id" => 1},
+				%{"login" => "u2", "id" => 2},
+				%{"login" => "a2", "id" => 12}
+			]
+		end
+
+		before do
+			result = Octograph.UserNodeRepo.find_or_create_followers(data)
+			{:ok, result: result}
+		end
+
+		let :records, do: Octograph.UserNodeRepo.find_by_logins(["u1", "u2", "a1", "a2"])
+
+		it "checks records" do
+			records	|> should have_count 4
+		end
+
+		it "checks result" do
+		  __.result
+		  |> Enum.map(fn(r) -> r.login end)
+		  |> should eq(["u1", "u2", "a1", "a2"])
+		end
+
+
 	end
 end

web/controllers/page_controller.ex

@@ -4,7 +4,7 @@ defmodule Octograph.PageController do
   plug :action
 
   def index(conn, _params) do
-    {nodes, edges} = Octograph.Traverser.level(6)
+    {nodes, edges} = Octograph.Traverser.level(3)
     {nodes, edges} = {Enum.uniq(nodes), Enum.uniq(edges)}
     render conn, "index.html", nodes: nodes, edges: edges
   end

web/controllers/user_nodes_controller.ex

@@ -6,6 +6,7 @@ defmodule Octograph.UserNodesController do
   def index(conn, _params) do
     user_node = Octograph.UserNodeRepo.last
     count = Octograph.UserNodeRepo.count
-    render conn, "index.html", user_node: user_node, count: count
+    edges_count = Octograph.UserNodeRepo.count
+    render conn, "index.html", user_node: user_node, count: count, edges_count: edges_count
   end
 end

web/models/user_node.ex

@@ -4,6 +4,8 @@ defmodule Octograph.UserNode do
 	schema "user_nodes" do
     field :login, :string
     field :github_id, :integer
+    field :flrs_count, :integer
+  	field :flng_count, :integer
     field :followers_checked_at, Ecto.DateTime
   end
 

web/repos/base_repo.ex

@@ -31,7 +31,7 @@ defmodule Octograph.BaseRepo do
 
       def count do
         query = from(un in module, select: count(un.id))
-        Octograph.Repo.one(query)
+        Octograph.Repo.one(query, timeout: :infinity)
       end
 
       def find_by_ids(ids) do

web/repos/user_node_repo.ex

@@ -3,16 +3,18 @@ defmodule Octograph.UserNodeRepo do
 
 	def find_by_login(login) do
 		query = from un in Octograph.UserNode, where: un.login == ^login, select: un
-    Octograph.Repo.all(query) |> List.first
+    Octograph.Repo.one(query)
 	end
 
-	def update_or_create(user_node) do
+	def find_by_logins(logins) do
+    query = from r in module, where: r.login in ^logins, select: r
+    Octograph.Repo.all(query)
+  end
+
+
+	def find_or_create(user_node) do
 		user = find_by_login(user_node.login)
-		if user do
-			user = update(%{user_node | id: user.id})
-		else
-			user = create(user_node)
-		end
+		unless user, do: user = create(user_node)
 		user
 	end
 
@@ -27,10 +29,27 @@ defmodule Octograph.UserNodeRepo do
     Octograph.Repo.one(query)
   end
 
-  def without_followers do
-		query = from(un in module, where: is_nil(un.followers_checked_at), select: un, limit: 100)
-    Octograph.Repo.all(query)
+  def with_max_followers do
+    query = from(un in module, where: not is_nil(un.flrs_count), order_by: [desc: :flrs_count], limit: 1)
+    Octograph.Repo.one(query, timeout: :infinity)
+  end
+
+  def first_without_followers do
+		query = from(un in module, where: is_nil(un.followers_checked_at), select: un, limit: 1)
+    Octograph.Repo.one(query)
+  end
+
+  def find_or_create_followers(data) do
+  	logins = Enum.map(data, fn(d) -> d["login"] end)
+   	stored = find_by_logins(logins)
+  	new = Enum.filter(data, fn(d) ->
+  		!Enum.any?(stored, fn(u) -> u.login == d["login"] end)
+  	end)
+  	Enum.each(new, fn(n) -> create(Octograph.UserNode.new(n))	end)
+  	find_by_logins(logins)
   end
-
+
+
+
 
 end

web/router.ex

@@ -18,10 +18,13 @@ defmodule Octograph.Router do
     get "/", PageController, :index
 
     resources "/user_nodes", UserNodesController, only: [:index]
-    resource "user_spider", UserSpiderController, only: [:show] do
-      post "/start", UserSpiderController, :start, as: :start
-      post "/stop", UserSpiderController, :stop, as: :stop
+
+    resources "/spiders", SpidersController, only: [] do
+      get "/",  SpidersController, :show
+      post "/start", SpidersController, :start, as: :start
+      post "/stop", SpidersController, :stop, as: :stop
     end
+
   end
 
   # Other scopes may use custom stacks.

web/templates/layout/application.html.eex

@@ -33,31 +33,19 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
           </button>
-          <a class="navbar-brand" href="#">Project name</a>
+          <a class="navbar-brand" href="/">Octograpth</a>
         </div>
         <div id="navbar" class="navbar-collapse collapse">
           <ul class="nav navbar-nav">
-            <li class="active"><a href="#">Home</a></li>
-            <li><a href="#about">About</a></li>
-            <li><a href="#contact">Contact</a></li>
+            <li><a href="<%= user_nodes_path(@conn, :index) %>">UserNodes</a></li>
             <li class="dropdown">
-              <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Dropdown <span class="caret"></span></a>
+              <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Spiders <span class="caret"></span></a>
               <ul class="dropdown-menu" role="menu">
-                <li><a href="#">Action</a></li>
-                <li><a href="#">Another action</a></li>
-                <li><a href="#">Something else here</a></li>
-                <li class="divider"></li>
-                <li class="dropdown-header">Nav header</li>
-                <li><a href="#">Separated link</a></li>
-                <li><a href="#">One more separated link</a></li>
+                <li><a href="<%= spiders_spiders_path(@conn, :show, :users) %>">Users</a></li>
+                <li><a href="<%= spiders_spiders_path(@conn, :show, :followers) %>">Followes</a></li>
               </ul>
             </li>
           </ul>
-          <ul class="nav navbar-nav navbar-right">
-            <li><a href="../navbar/">Default</a></li>
-            <li class="active"><a href="./">Static top <span class="sr-only">(current)</span></a></li>
-            <li><a href="../navbar-fixed-top/">Fixed top</a></li>
-          </ul>
         </div><!--/.nav-collapse -->
       </div>
     </nav>