Skip to content
Browse files

Filtering github events based on repo programming languages.

  • Loading branch information...
1 parent 383d1d5 commit 399c57f2ae10a3cb0864e9655d158f273b489e33 @rambocoder committed May 9, 2012
Showing with 151 additions and 28 deletions.
  1. +99 −28 github.erl
  2. BIN hello.sqlite
  3. +36 −0 prettyJSON.py
  4. +16 −0 useful snippets.txt
View
127 github.erl
@@ -1,38 +1,109 @@
-module(github).
--export([start/0, loop/1]).
+-export([start/0, stop/0, loop/2, repo/0]).
start() ->
crypto:start(),
inets:start(),
ssl:start(),
- sqlite3:open(ct, [{file, "/home/alex/Downloads/erlanghub/erlang-sqlite3/hello.sqlite"}]),
- loop("1").
-
-loop(PageNumber) ->
- timer:sleep(1000),
- InitialUrl = "https://api.github.com/events?page=",
- NextUrl = string:concat(InitialUrl, PageNumber),
- {ok, {Status1, Headers1, Body1}} = httpc:request(NextUrl),
+ Pid = erlang:spawn_link(fun github:repo/0),
+ register(repo_server, Pid),
+ sqlite3:open(ct, [{file, "/home/alex/Downloads/erlang_github/hello.sqlite"}]),
+ loop(1, queue:from_list(lists:seq(0, 3000))).
+
+stop() ->
+ repo_server ! stop,
+ unregister(repo_server),
+ sqlite3:close(ct).
+
+repo() ->
+ receive
+ stop -> ok;
+ {RepoId, RepoUrl} ->
+ % check if the repo exists in the database
+ case is_number(RepoId) of
+ false -> repo();
+ true -> continue
+ end,
+ Result = sqlite3:sql_exec(ct, "SELECT * FROM repos WHERE id = ?", [RepoId]),
+ case length(proplists:get_value(rows, Result)) of
+ 1 -> repo();
+ _ -> not_found
+ end,
+ LanguageUrl = string:concat(unicode:characters_to_list(RepoUrl), "/languages"),
+ io:format("Language URL: ~p~n", [LanguageUrl]),
+ io:format("Repo id: ~p~n", [RepoId]),
+ {ok, {{_Version, StatusCode, _Reason}, Headers1, Body1}} = httpc:request(LanguageUrl),
+ case StatusCode of
+ 200 ->
+ {Languages} = jiffy:decode(Body1),
+ Erlang = case proplists:is_defined(<<"Erlang">>, Languages) of
+ true -> "TRUE";
+ false -> "FALSE"
+ end,
+
+ sqlite3:sql_exec(ct, "INSERT INTO repos (id, URL, Erlang, JSON) VALUES (?, ?, ?, ?)",
+ [RepoId,
+ RepoUrl,
+ Erlang,
+ Body1]);
+ _ -> nothing
+ end,
+ timer:sleep(1000),
+ repo()
+ end.
+
+loop(Iteration, S) when Iteration < 20 ->
+ % timer:sleep(1000),
+ Url = "https://api.github.com/events",
+ {ok, {Status1, Headers1, Body1}} = httpc:request(Url),
% process the content of JSON data
- Obj = mochijson2:decode(Body1),
- Events = proplists:get_all_values(struct, Obj),
- lists:foreach(fun(Event) -> insert_event(Event) end, Events),
+ case catch jiffy:decode(Body1) of
+ {'EXIT', Reason } -> io:format("Caught: ~p~n", [Reason]), Events = [], timer:sleep(10000), loop(Iteration + 1, S);
+ Events -> Events
+ end,
+ S3 = lists:foldl(fun(Event, Acc) -> insert_event(Event, Iteration, Acc) end, S, lists:reverse(Events)),
+
% ok = file:write_file("output.txt", Body1, [write, append]),
- Link = proplists:get_value("link", Headers1),
- case string:str(Link, "next") of
- 0 -> io:format("Start from page 1~n"),
- loop("1");
- _ -> LinkStart = string:str(Link, "=") + 1,
- LinkEnd = string:str(Link, ">") -1,
- NextPageNumber = string:sub_string(Link, LinkStart, LinkEnd),
- io:format("Next page: ~p~n", [NextPageNumber]),
- RemainingHits = proplists:get_value("x-ratelimit-remaining", Headers1),
- io:format("Remaining hits: ~p~n", [RemainingHits]),
- loop(NextPageNumber)
+ RemainingHits = proplists:get_value("x-ratelimit-remaining", Headers1),
+ io:format("Remaining hits: ~p~n", [RemainingHits]),
+ io:format("Start ~p iteration~n", [Iteration]),
+ timer:sleep(10000),
+ loop(Iteration + 1, S3);
+
+loop(_, _) ->
+ stop.
+
+insert_event(Event, Iteration, Acc) ->
+ {E} = Event,
+ EventId = proplists:get_value(<<"id">>, E),
+ case queue:member(EventId, Acc) of
+ true -> Acc;
+ false -> sqlite3:sql_exec(ct, "INSERT INTO github5 (id, github_time, pull_time, iteration) VALUES (?, ?, ?, ?)",
+ [EventId,
+ proplists:get_value(<<"created_at">>, E),
+ httpd_util:rfc1123_date(erlang:localtime()),
+ Iteration]),
+ {Repo} = proplists:get_value(<<"repo">>, E),
+ RepoUrl = proplists:get_value(<<"url">>, Repo),
+ RepoId = proplists:get_value(<<"id">>, Repo),
+ repo_server ! {RepoId, RepoUrl},
+ % check if the event for a possible erlang repo
+ check_repo(RepoId, E),
+
+
+ Q1 = queue:drop(Acc), % remove the first element from the que % add as the last element to the que
+ queue:snoc(Q1, EventId)
end.
-insert_event(Event) ->
- io:format("~p~n", [proplists:get_value(<<"id">>, Event)]),
- io:format("~p~n", [proplists:get_value(<<"created_at">>, Event)]),
- sqlite3:sql_exec(ct, "INSERT INTO github (id, github_time, pull_time) VALUES (?, ?, ?)",
- [proplists:get_value(<<"id">>, Event), proplists:get_value(<<"created_at">>, Event), httpd_util:rfc1123_date(erlang:localtime())]).
+check_repo(RepoId, E) when is_number(RepoId) ->
+ Result = sqlite3:sql_exec(ct, "SELECT erlang FROM repos WHERE id = ?", [RepoId]),
+ Rows = proplists:get_value(rows, Result),
+ publish(Rows, E);
+check_repo(_, _) -> nothing.
+
+publish([], _) ->
+ nothing;
+publish([{<<"TRUE">>}], E) ->
+ io:format("Erlang event ~p~n", [E]);
+publish([{<<"FALSE">>}], _) ->
+ nothing.
View
BIN hello.sqlite
Binary file not shown.
View
36 prettyJSON.py
@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+"""
+Convert JSON data to human-readable form.
+
+Usage:
+ prettyJSON.py inputFile [outputFile]
+"""
+
+import sys
+import simplejson as json
+
+
+def main(args):
+ try:
+ inputFile = open(args[1])
+ input = json.load(inputFile)
+ inputFile.close()
+ except IndexError:
+ usage()
+ return False
+ if len(args) < 3:
+ print json.dumps(input, sort_keys = False, indent = 4)
+ else:
+ outputFile = open(args[2], "w")
+ json.dump(input, outputFile, sort_keys = False, indent = 4)
+ outputFile.close()
+ return True
+
+
+def usage():
+ print __doc__
+
+
+if __name__ == "__main__":
+ sys.exit(not main(sys.argv))
View
16 useful snippets.txt
@@ -0,0 +1,16 @@
+{ok, {Status1, Headers1, Body1}} = httpc:request("https://api.github.com/events").
+{Events} = jiffy:decode(Body1).
+
+
+{ok, {Status1, Headers1, Body1}} = httpc:request("https://api.github.com/repos/depesz/docbot/languages").
+
+sqlite3:open(ct, [{file, "/home/alex/Downloads/erlang_github/hello.sqlite"}]).
+sqlite3:sql_exec(ct, "SELECT * FROM repos WHERE id = ?", [4258124]);
+
+
+
+38> sqlite3:sql_exec(ct, "SELECT erlang FROM repos WHERE id = ?", [33]).
+[{columns,["(ä`³ng"]},{rows,[]}]
+39> sqlite3:sql_exec(ct, "SELECT erlang FROM repos WHERE id = ?", [4258124]).
+[{columns,["Erlang"]},{rows,[{<<"FALSE">>}]}]
+

0 comments on commit 399c57f

Please sign in to comment.
Something went wrong with that request. Please try again.