Skip to content

Commit 1702115

Browse files
chewbrancachewbranca
committed
WIP: hack together filtered reporting logic
1 parent 8d0e9fd commit 1702115

File tree

2 files changed

+230
-2
lines changed

2 files changed

+230
-2
lines changed

‎src/couch_stats/src/couch_stats_resource_tracker.erl

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414

1515
-behaviour(gen_server).
1616

17+
-export([
18+
new/0,
19+
tnow/0
20+
]).
21+
1722
-export([
1823
start_link/0,
1924
init/1,
@@ -153,8 +158,8 @@
153158

154159
-record(rctx, {
155160
%% Metadata
156-
started_at = tnow(),
157-
updated_at = tnow(),
161+
started_at = ?MODULE:tnow(),
162+
updated_at = ?MODULE:tnow(),
158163
pid_ref,
159164
mfa,
160165
nonce,
@@ -193,6 +198,8 @@
193198
%% PidRef operations
194199
%%
195200

201+
new() -> #rctx{}.
202+
196203
get_pid_ref() ->
197204
get(?PID_REF).
198205

‎src/couch_stats/src/csrt_logger.erl

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
2+
% use this file except in compliance with the License. You may obtain a copy of
3+
% the License at
4+
%
5+
% http://www.apache.org/licenses/LICENSE-2.0
6+
%
7+
% Unless required by applicable law or agreed to in writing, software
8+
% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9+
% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10+
% License for the specific language governing permissions and limitations under
11+
% the License.
12+
13+
-module(csrt_logger).
14+
15+
-behaviour(gen_server).
16+
17+
-export([
18+
start_link/0,
19+
init/1,
20+
handle_call/3,
21+
handle_cast/2
22+
]).
23+
24+
-export([
25+
log_level/0,
26+
log_level/1,
27+
maybe_report/1
28+
]).
29+
30+
-record(st, {
31+
should_track,
32+
matchers = #{}
33+
}).
34+
35+
%% TODO: switch to include for record def
36+
-record(rctx, {
37+
%% Metadata
38+
started_at,
39+
updated_at,
40+
pid_ref,
41+
mfa,
42+
nonce,
43+
from,
44+
type = unknown, %% unknown/background/system/rpc/coordinator/fabric_rpc/etc_rpc/etc
45+
dbname,
46+
username,
47+
path,
48+
49+
%% Stats counters
50+
db_open = 0,
51+
docs_read = 0,
52+
rows_read = 0,
53+
changes_processed = 0,
54+
changes_returned = 0,
55+
ioq_calls = 0,
56+
io_bytes_read = 0,
57+
io_bytes_written = 0,
58+
js_evals = 0,
59+
js_filter = 0,
60+
js_filtered_docs = 0,
61+
mango_eval_match = 0,
62+
%% TODO: switch record definitions to be macro based, eg:
63+
%% ?COUCH_BT_GET_KP_NODE = 0,
64+
get_kv_node = 0,
65+
get_kp_node = 0,
66+
write_kv_node = 0,
67+
write_kp_node = 0
68+
}).
69+
70+
-record(rpc_worker, {
71+
mod,
72+
func
73+
}).
74+
75+
-record(coordinator, {}).
76+
77+
-define(CSRT_CONTEXTS, [
78+
#rpc_worker{mod=fabric_rpc, func=all_docs},
79+
#rpc_worker{mod=fabric_rpc, func=changes},
80+
#rpc_worker{mod=fabric_rpc, func=map_view},
81+
#rpc_worker{mod=fabric_rpc, func=reduce_view},
82+
#rpc_worker{mod=fabric_rpc, func=get_all_security},
83+
#rpc_worker{mod=fabric_rpc, func=open_doc},
84+
#rpc_worker{mod=fabric_rpc, func=update_docs},
85+
#rpc_worker{mod=fabric_rpc, func=open_shard},
86+
#rpc_worker{mod=mango_cursor, func=view}
87+
%% enable all: #worker{}
88+
%% TODO: add coordinator/etc
89+
%% #coordinator{mod=chttpd_db:}
90+
]).
91+
92+
-define(CSRT_LOG_LEVELS, [
93+
{all, 9},
94+
{workers_too, 4},
95+
{custom, 2},
96+
{coordinators, 1},
97+
{none, 0}
98+
]).
99+
100+
-define(NO_LOG, 0).
101+
102+
log_level(all) -> 9;
103+
log_level(all_workers) -> 4;
104+
log_level(rpc_workers) -> 3;
105+
log_level(#rctx{type=#rpc_worker{}}) -> 3;
106+
log_level(custom) -> 2;
107+
log_level(coordinators) -> 1;
108+
log_level(#rctx{type=#coordinator{}}) -> 1;
109+
log_level(none) -> ?NO_LOG;
110+
log_level(_) -> ?NO_LOG.
111+
112+
log_level() ->
113+
config:get_atom("couch_stats_resource_tracker", "log_level", none).
114+
115+
maybe_report(PidRef) ->
116+
Level = log_level(),
117+
case Level > ?NO_LOG of
118+
true ->
119+
Rctx = couch_stats_resource_tracker:get_resource(PidRef),
120+
RLevel = log_level(Rctx),
121+
case RLevel =< Level of
122+
true ->
123+
%% NOTE: we drop the PidRef here, it's now safe to delete the ets entry
124+
Msg = {maybe_report, Rctx, RLevel, Level},
125+
gen_server:cast(get_server(), Msg);
126+
false ->
127+
ok
128+
end;
129+
false ->
130+
ok
131+
end.
132+
133+
find_matches(RContexts, Matchers) when is_list(RContexts) andalso is_map(Matchers) ->
134+
maps:filter(
135+
fun(_Name, {_MSpec, CompMSpec}) ->
136+
catch [] =/= ets:match_spec_run(RContexts, CompMSpec)
137+
end,
138+
Matchers
139+
).
140+
141+
is_match(#rctx{}=R, Matchers) when is_map(Matchers) ->
142+
maps:size(find_matches([R], Matchers)) > 0.
143+
144+
%% default.ini:
145+
%% [csrt]
146+
%% enabled = false
147+
%%
148+
%% [csrt.should_log]
149+
%% worker.fabric_rpc.all_docs = true
150+
%% worker.fabric_rpc.map_view = true
151+
%% worker.fabric_rpc.reduce_view = true
152+
create_tracking_map() ->
153+
lists:fold(fun(Ele, Acc) ->
154+
Key = case Ele of
155+
#rpc_worker{mod=M0, func=F0} ->
156+
M = atom_to_list(M0),
157+
F = atom_to_list(F0),
158+
"worker." ++ M ++ "." ++ F
159+
%% TODO: #coordinator{} ->
160+
end,
161+
Enabled = config:get_boolean("csrt.should_log", Key, true),
162+
maps:put(Ele, Enabled, Acc)
163+
end, #{}, ?CSRT_CONTEXTS).
164+
165+
start_link() ->
166+
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
167+
168+
init([]) ->
169+
{ok, #st{should_track=create_tracking_map()}}.
170+
171+
handle_call({register, Name, MSpec}, _From, #st{matchers=Matchers}=St) ->
172+
case add_matcher(Name, MSpec, Matchers) of
173+
{ok, Matchers1} ->
174+
{reply, ok, St#st{matchers=Matchers1}};
175+
{error, badarg}=Error ->
176+
{reply, Error, St}
177+
end;
178+
handle_call({maybe_report, #rctx{}=Rctx, RLevel, Level}, _From, St) ->
179+
ok = maybe_report_int(Rctx, RLevel, Level, St),
180+
{reply, ok, St};
181+
handle_call(_, _From, State) ->
182+
{reply, ok, State}.
183+
184+
handle_cast(_Msg, State) ->
185+
{noreply, State, 0}.
186+
187+
%% TODO: should we use originally defined log level? or St#st.level?
188+
%% ShouldLog = log_level(Rctx) > St#st.log_level
189+
maybe_report_int(#rctx{}=Rctx, RLevel, Level, #st{}) when RLevel >= Level ->
190+
report(Rctx);
191+
maybe_report_int(#rctx{}=Rctx, _RLevel, _Level, #st{matchers=Matchers, should_track=Trackers}) ->
192+
case is_tracked(Rctx, Trackers) orelse is_match(Rctx, Matchers) of
193+
true ->
194+
report(Rctx);
195+
false ->
196+
ok
197+
end.
198+
199+
report(Rctx) ->
200+
couch_log:report(
201+
"csrt-pid-usage-lifetime",
202+
couch_stats_resource_tracker:to_json(Rctx)
203+
).
204+
205+
get_server() ->
206+
%% TODO: shard servers like couch_server/etc
207+
?MODULE.
208+
209+
add_matcher(Name, MSpec, Matchers) ->
210+
try ets:match_spec_compile(MSpec) of
211+
CompMSpec ->
212+
%% TODO: handle already registered name case
213+
Matchers1 = maps:put(Name, {MSpec, CompMSpec}, Matchers),
214+
{ok, Matchers1}
215+
catch
216+
error:badarg ->
217+
{error, badarg}
218+
end.
219+
220+
is_tracked(#rctx{type=Type}, Trackers) ->
221+
maps:is_key(Type, Trackers) andalso true =:= maps:get(Type, Trackers).

0 commit comments

Comments
 (0)