Permalink
Browse files

merging nsync into master

  • Loading branch information...
2 parents 4f95cbf + f44f9a9 commit 74dc7e0ff079bc01d5fde04c3428b10bc50771d3 @jkvor jkvor committed Jun 16, 2011
View
@@ -1,4 +1,8 @@
release/*.boot
ebin/*.beam
erl_crash.dump
-go_src/*.out
+data
+channels
+tokens
+drains
+src/*.swp
View
@@ -7,3 +7,12 @@
[submodule "deps/pagerduty"]
path = deps/pagerduty
url = git@git.herokai.com:erlang_pagerduty.git
+[submodule "deps/redgrid"]
+ path = deps/redgrid
+ url = git@git.herokai.com:redgrid.git
+[submodule "deps/redo"]
+ path = deps/redo
+ url = git@git.herokai.com:redo.git
+[submodule "deps/nsync"]
+ path = deps/nsync
+ url = git@git.herokai.com:nsync.git
View
@@ -2,6 +2,9 @@ all:
(cd deps/redis_pool;$(MAKE) all)
(cd deps/mochiweb;$(MAKE) all)
(cd deps/pagerduty;$(MAKE) all)
+ (cd deps/redgrid;$(MAKE) all)
+ (cd deps/redo;$(MAKE) all)
+ (cd deps/nsync;$(MAKE) all)
@erl -make
@escript release/build_rel.escript boot logplex `pwd`/ebin
@@ -12,3 +15,6 @@ clean_all: clean
(cd deps/redis_pool;$(MAKE) clean)
(cd deps/mochiweb;$(MAKE) clean)
(cd deps/pagerduty;$(MAKE) clean)
+ (cd deps/redgrid;$(MAKE) clean)
+ (cd deps/redo;$(MAKE) clean)
+ (cd deps/nsync;$(MAKE) clean)
View
@@ -8,4 +8,4 @@ else
fi
ulimit -n 65535
-erl +K true +A30 +P500000 -env ERL_MAX_PORTS 65535 -env HTTP_PORT 8002 -env INSTANCE_NAME localhost -env LOGPLEX_AUTH_KEY secret -env LOGPLEX_WORKERS 1 -env LOGPLEX_DRAIN_WRITERS 1 -env LOGPLEX_REDIS_WRITERS 1 -env LOGPLEX_READERS 1 -name logplex_console@$hostname -pa ebin -pa deps/*/ebin -boot release/logplex-1.0
+erl +K true +A30 +P500000 -env ERL_MAX_PORTS 65535 -env HTTP_PORT 8002 -env INSTANCE_NAME localhost -env LOGPLEX_AUTH_KEY secret -env LOGPLEX_WORKERS 1 -env LOGPLEX_DRAIN_WRITERS 1 -env LOGPLEX_REDIS_WRITERS 1 -env LOGPLEX_READERS 1 -name logplex_console@$hostname -pa ebin -pa deps/*/ebin -boot release/logplex-1.0
View
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+if [ $# -ne 1 ]
+then
+ echo "Usage: `basename $0` {node}"
+ exit 1
+fi
+
+HOSTNAME=`hostname --fqdn`
+COOKIE=`awk -F"'" '/LOGPLEX_COOKIE/ { print $2 }' /home/logplex/keys.sh`
+HOME=/home/logplex
+
+erl -name recover@$HOSTNAME -pa ebin -pa deps/*/ebin -setcookie $COOKIE -noinput -eval "logplex_utils:rpc('logplex@$HOSTNAME', 'logplex_db', 'recover_from', ['$1'])" -s init stop
+
View
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-envdir keys erl -pa ebin -pa deps/*/ebin -noinput -eval "logplex_utils:shard_info()" -s init stop
View
@@ -1,4 +1,7 @@
#!/bin/sh
-. /home/logplex/keys.sh
-erl -pa ebin -pa deps/*/ebin -noinput -eval "logplex_utils:set_weight($1)" -s init stop
+HOSTNAME=`hostname --fqdn`
+COOKIE=`awk -F"'" '/LOGPLEX_COOKIE/ { print $2 }' /home/logplex/keys.sh`
+HOME=/home/logplex
+
+erl -name nodes@$HOSTNAME -pa ebin -pa deps/*/ebin -hidden -setcookie $COOKIE -noinput -eval "logplex_utils:rpc('logplex@$HOSTNAME', 'logplex_utils', 'set_weight', [$1])" -s init stop
Submodule nsync added at e4f34b
Submodule redgrid added at 5ff5f6
Submodule redo added at c612f2
View
@@ -6,12 +6,9 @@
logplex_api,
logplex_app,
logplex_channel,
- logplex_cloudkick,
logplex_drain,
logplex_drain_writer,
- logplex_grid,
logplex_queue_sup,
- logplex_rate_limit,
logplex_realtime,
logplex_redis_writer,
logplex_session,
View
@@ -1,20 +1,17 @@
-include_lib("eunit/include/eunit.hrl").
-record(msg, {time, source, ps, content}).
--record(channel, {id, name, app_id, addon}).
--record(token, {id, channel_id, name, app_id, addon}).
--record(drain, {id, channel_id, resolved_host, host, port}).
+-record(channel, {id, name, app_id}).
+-record(token, {id, channel_id, name, app_id, drains=[]}).
+-record(drain, {id, channel_id, token, resolved_host, host, port}).
+-record(session, {id, body}).
-define(HTTP_PORT, 8001).
-define(TCP_PORT, 9998).
-define(UDP_PORT, 9999).
--define(DEFAULT_LOG_HISTORY, <<"500">>).
--define(ADVANCED_LOG_HISTORY, <<"1500">>).
-
--define(BASIC_THROUGHPUT, 500).
--define(EXPANDED_THROUGHPUT, 10000).
-
+-define(MAX_DRAINS, 5).
+-define(LOG_HISTORY, <<"1500">>).
-define(MAX_SPOOL_POOL_SIZE, 1000).
-define(DEFAULT_LOGPLEX_QUEUE_LENGTH, 2000).
View
@@ -59,14 +59,16 @@ loop(Req) ->
{Code, Body} = serve(handlers(), Method, Path, Req),
Time = timer:now_diff(now(), Start) div 1000,
io:format("logplex_api app_id=~s channel_id=~s method=~p path=~s resp_code=~w time=~w body=~s~n", [AppId, ChannelId, Method, Path, Code, Time, Body]),
- Req:respond({Code, ?HDR, Body})
+ Req:respond({Code, ?HDR, Body}),
+ exit(normal)
catch
exit:normal ->
- ok;
+ exit(normal);
Class:Exception ->
Time1 = timer:now_diff(now(), Start) div 1000,
io:format("logplex_api app_id=~s channel_id=~s method=~p path=~s time=~w exception=~1000p:~1000p~n", [AppId, ChannelId, Method, Path, Time1, Class, Exception]),
- Req:respond({500, ?HDR, ""})
+ Req:respond({500, ?HDR, ""}),
+ exit(normal)
end.
handlers() ->
@@ -75,7 +77,7 @@ handlers() ->
[throw({500, io_lib:format("Zero ~p child processes running", [Worker])}) || {Worker, 0} <- logplex_stats:workers()],
- RegisteredMods = [logplex_grid, logplex_rate_limit, logplex_realtime, logplex_stats, logplex_channel, logplex_token, logplex_drain, logplex_session, logplex_tail, logplex_shard, udp_acceptor],
+ RegisteredMods = [logplex_realtime, logplex_stats, logplex_tail, logplex_shard, udp_acceptor],
[(whereis(Name) == undefined orelse not is_process_alive(whereis(Name))) andalso throw({500, io_lib:format("Process dead: ~p", [Name])}) || Name <- RegisteredMods],
Count = logplex_stats:healthcheck(),
@@ -85,19 +87,18 @@ handlers() ->
end},
{['POST', "/channels$"], fun(Req, _Match) ->
+ readonly(Req),
authorize(Req),
- {struct, Params} = mochijson2:decode(Req:recv_body()),
+ Body = Req:recv_body(),
+ {struct, Params} = mochijson2:decode(Body),
ChannelName = proplists:get_value(<<"name">>, Params),
ChannelName == undefined andalso error_resp(400, <<"'name' post param missing">>),
AppId = proplists:get_value(<<"app_id">>, Params),
AppId == undefined andalso error_resp(400, <<"'app_id' post param missing">>),
- Addon = proplists:get_value(<<"addon">>, Params),
- Addon == undefined andalso error_resp(400, <<"'addon' post param missing">>),
-
- ChannelId = logplex_channel:create(ChannelName, AppId, Addon),
+ ChannelId = logplex_channel:create(ChannelName, AppId),
not is_integer(ChannelId) andalso exit({expected_integer, ChannelId}),
case proplists:get_value(<<"tokens">>, Params) of
@@ -113,20 +114,14 @@ handlers() ->
end
end},
- {['POST', "/channels/(\\d+)/addon$"], fun(Req, [ChannelId]) ->
+ {['POST', "/channels/(\\d+)/addon$"], fun(Req, [_ChannelId]) ->
+ readonly(Req),
authorize(Req),
- {struct, Params} = mochijson2:decode(Req:recv_body()),
-
- Addon = proplists:get_value(<<"addon">>, Params),
- Addon == undefined andalso error_resp(400, <<"'addon' post param missing">>),
-
- case logplex_channel:update_addon(list_to_integer(ChannelId), Addon) of
- ok -> {200, <<"OK">>};
- {error, not_found} -> {404, <<"Not found">>}
- end
+ {200, <<"OK">>}
end},
{['DELETE', "/channels/(\\d+)$"], fun(Req, [ChannelId]) ->
+ readonly(Req),
authorize(Req),
case logplex_channel:delete(list_to_integer(ChannelId)) of
ok -> {200, <<"OK">>};
@@ -135,15 +130,21 @@ handlers() ->
end},
{['POST', "/channels/(\\d+)/token$"], fun(Req, [ChannelId]) ->
+ readonly(Req),
authorize(Req),
{struct, Params} = mochijson2:decode(Req:recv_body()),
TokenName = proplists:get_value(<<"name">>, Params),
TokenName == undefined andalso error_resp(400, <<"'name' post param missing">>),
+ A = now(),
Token = logplex_token:create(list_to_integer(ChannelId), TokenName),
+ B = now(),
not is_binary(Token) andalso exit({expected_binary, Token}),
+ io:format("create_token name=~s channel_id=~s time=~w~n",
+ [TokenName, ChannelId, timer:now_diff(B,A) div 1000]),
+
{201, Token}
end},
@@ -156,6 +157,8 @@ handlers() ->
end},
{['GET', "/sessions/([\\w-]+)$"], fun(Req, [Session]) ->
+ proplists:get_value("srv", Req:parse_qs()) == undefined
+ andalso error_resp(400, <<"[Error]: Please update your Heroku client to the most recent version\n">>),
Body = logplex_session:lookup(list_to_binary("/sessions/" ++ Session)),
not is_binary(Body) andalso error_resp(404, <<"Not found">>),
@@ -205,6 +208,7 @@ handlers() ->
end},
{['POST', "/channels/(\\d+)/drains$"], fun(Req, [ChannelId]) ->
+ readonly(Req),
authorize(Req),
{struct, Data} = mochijson2:decode(Req:recv_body()),
@@ -214,27 +218,33 @@ handlers() ->
Host == <<"localhost">> andalso error_resp(400, <<"Invalid drain">>),
Host == <<"127.0.0.1">> andalso error_resp(400, <<"Invalid drain">>),
- DrainId = logplex_drain:create(list_to_integer(ChannelId), Host, Port),
- case DrainId of
- Int when is_integer(Int) ->
- {201, io_lib:format("Successfully added drain syslog://~s:~p", [Host, Port])};
- {error, already_exists} ->
- {400, io_lib:format("Drain syslog://~s:~p already exists", [Host, Port])};
- {error, invalid_drain} ->
- {400, io_lib:format("Invalid drain syslog://~s:~p", [Host, Port])}
- end
+ case logplex_channel:lookup_drains(list_to_integer(ChannelId)) of
+ List when length(List) >= ?MAX_DRAINS ->
+ {400, "You have already added the maximum number of drains allowed"};
+ _ ->
+ case logplex_drain:create(list_to_integer(ChannelId), Host, Port) of
+ #drain{id=_Id, token=_Token} ->
+ {201, io_lib:format("Successfully added drain syslog://~s:~p", [Host, Port])};
+ %{201, iolist_to_binary(mochijson2:encode({struct, [{id, Id}, {token, Token}]}))};
+ {error, already_exists} ->
+ {400, io_lib:format("Drain syslog://~s:~p already exists", [Host, Port])};
+ {error, invalid_drain} ->
+ {400, io_lib:format("Invalid drain syslog://~s:~p", [Host, Port])}
+ end
+ end
end},
{['GET', "/channels/(\\d+)/drains$"], fun(Req, [ChannelId]) ->
authorize(Req),
- Drains = logplex_channel:drains(list_to_integer(ChannelId)),
+ Drains = logplex_channel:lookup_drains(list_to_integer(ChannelId)),
not is_list(Drains) andalso exit({expected_list, Drains}),
Drains1 = [{struct, [{host, Host}, {port, Port}]} || #drain{host=Host, port=Port} <- Drains],
{200, iolist_to_binary(mochijson2:encode(Drains1))}
end},
{['DELETE', "/channels/(\\d+)/drains$"], fun(Req, [ChannelId]) ->
+ readonly(Req),
authorize(Req),
Data = Req:parse_qs(),
@@ -287,6 +297,13 @@ authorize(Req) ->
throw({401, <<"Not Authorized">>})
end.
+readonly(_Req) ->
+ case application:get_env(logplex, read_only) of
+ {ok, true} ->
+ throw({500, <<"Read-only mode">>});
+ _ -> ok
+ end.
+
error_resp(RespCode, Body) ->
error_resp(RespCode, Body, undefined).
@@ -320,6 +337,8 @@ tail_loop(Socket, Filters) ->
logplex_utils:filter(Msg1, Filters) andalso gen_tcp:send(Socket, logplex_utils:format(Msg1)),
tail_loop(Socket, Filters);
{tcp_closed, Socket} ->
+ ok;
+ {tcp_error, Socket, _Reason} ->
ok
end.
View
@@ -36,23 +36,22 @@ start(_StartType, _StartArgs) ->
set_cookie(),
boot_pagerduty(),
boot_redis(),
+ setup_redgrid_vals(),
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
stop(_State) ->
+ io:format("stopping...~n"),
+ logplex_db:dump(),
ok.
init([]) ->
{ok, {{one_for_one, 5, 10}, [
- {logplex_rate_limit, {logplex_rate_limit, start_link, []}, permanent, 2000, worker, [logplex_rate_limit]},
+ {logplex_db, {logplex_db, start_link, []}, permanent, 2000, worker, [logplex_db]},
+ {redgrid, {redgrid, start_link, []}, permanent, 2000, worker, [redgrid]},
{logplex_realtime, {logplex_realtime, start_link, [logplex_utils:redis_opts("LOGPLEX_CONFIG_REDIS_URL")]}, permanent, 2000, worker, [logplex_realtime]},
{logplex_stats, {logplex_stats, start_link, []}, permanent, 2000, worker, [logplex_stats]},
- {logplex_channel, {logplex_channel, start_link, []}, permanent, 2000, worker, [logplex_channel]},
- {logplex_token, {logplex_token, start_link, []}, permanent, 2000, worker, [logplex_token]},
- {logplex_drain, {logplex_drain, start_link, []}, permanent, 2000, worker, [logplex_drain]},
- {logplex_session, {logplex_session, start_link, []}, permanent, 2000, worker, [logplex_session]},
-
- {logplex_grid, {logplex_grid, start_link, []}, permanent, 2000, worker, [logplex_grid]},
+ {logplex_token, {logplex_token, refresh_dns, []}, permanent, 2000, worker, [logplex_token]},
{logplex_tail, {logplex_tail, start_link, []}, permanent, 2000, worker, [logplex_tail]},
{logplex_redis_writer_sup, {logplex_worker_sup, start_link, [logplex_redis_writer_sup, logplex_redis_writer]}, permanent, 2000, worker, [logplex_redis_writer_sup]},
@@ -67,7 +66,6 @@ init([]) ->
{logplex_work_queue, {logplex_queue, start_link, [logplex_work_queue, logplex_work_queue_args()]}, permanent, 2000, worker, [logplex_work_queue]},
{logplex_drain_buffer, {logplex_queue, start_link, [logplex_drain_buffer, logplex_drain_buffer_args()]}, permanent, 2000, worker, [logplex_drain_buffer]},
- {logplex_cloudkick, {logplex_cloudkick, start_link, []}, permanent, 2000, worker, [logplex_cloudkick]},
{logplex_api, {logplex_api, start_link, []}, permanent, 2000, worker, [logplex_api]},
{udp_acceptor, {udp_acceptor, start_link, []}, permanent, 2000, worker, [udp_acceptor]}
]}}.
@@ -81,22 +79,27 @@ set_cookie() ->
boot_pagerduty() ->
case os:getenv("HEROKU_DOMAIN") of
"heroku.com" ->
- ok = application:load(pagerduty),
- application:set_env(pagerduty, service_key, os:getenv("ROUTING_PAGERDUTY_SERVICE_KEY")),
- ok = application:start(pagerduty, temporary),
- ok = error_logger:add_report_handler(logplex_report_handler);
+ case os:getenv("PAGERDUTY") of
+ "0" -> ok;
+ _ ->
+ ok = application:load(pagerduty),
+ application:set_env(pagerduty, service_key, os:getenv("ROUTING_PAGERDUTY_SERVICE_KEY")),
+ ok = application:start(pagerduty, temporary),
+ ok = error_logger:add_report_handler(logplex_report_handler)
+ end;
_ ->
ok
end.
boot_redis() ->
- case application:start(redis, temporary) of
- ok ->
- Opts = logplex_utils:redis_opts("LOGPLEX_CONFIG_REDIS_URL"),
- redis_pool:add(config_pool, Opts, 25);
- Err ->
- exit(Err)
- end.
+ application:start(redis, temporary).
+
+setup_redgrid_vals() ->
+ application:load(redgrid),
+ application:set_env(redgrid, local_ip, os:getenv("LOCAL_IP")),
+ application:set_env(redgrid, redis_url, os:getenv("LOGPLEX_CONFIG_REDIS_URL")),
+ application:set_env(redgrid, domain, os:getenv("HEROKU_DOMAIN")),
+ ok.
logplex_work_queue_args() ->
MaxLength =
@@ -113,7 +116,8 @@ logplex_work_queue_args() ->
{max_length, MaxLength},
{num_workers, NumWorkers},
{worker_sup, logplex_worker_sup},
- {worker_args, []}].
+ {worker_args, []},
+ {dict, dict:from_list([{producer_callback, fun(Self, Atom) -> whereis(udp_acceptor) ! {Self, Atom} end}])}].
logplex_drain_buffer_args() ->
MaxLength =
Oops, something went wrong.

0 comments on commit 74dc7e0

Please sign in to comment.