Skip to content

Commit d25e37d

Browse files
rostedtIngo Molnar
rostedt
authored and
Ingo Molnar
committed
tracepoint: Optimize using static_call()
Currently the tracepoint site will iterate a vector and issue indirect calls to however many handlers are registered (ie. the vector is long). Using static_call() it is possible to optimize this for the common case of only having a single handler registered. In this case the static_call() can directly call this handler. Otherwise, if the vector is longer than 1, call a function that iterates the whole vector like the current code. [peterz: updated to new interface] Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: https://lore.kernel.org/r/20200818135805.279421092@infradead.org
1 parent a945c83 commit d25e37d

File tree

4 files changed

+94
-36
lines changed

4 files changed

+94
-36
lines changed

‎include/linux/tracepoint-defs.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include <linux/atomic.h>
1212
#include <linux/static_key.h>
1313

14+
struct static_call_key;
15+
1416
struct trace_print_flags {
1517
unsigned long mask;
1618
const char *name;
@@ -30,6 +32,9 @@ struct tracepoint_func {
3032
struct tracepoint {
3133
const char *name; /* Tracepoint name */
3234
struct static_key key;
35+
struct static_call_key *static_call_key;
36+
void *static_call_tramp;
37+
void *iterator;
3338
int (*regfunc)(void);
3439
void (*unregfunc)(void);
3540
struct tracepoint_func __rcu *funcs;

‎include/linux/tracepoint.h‎

Lines changed: 61 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include <linux/cpumask.h>
2020
#include <linux/rcupdate.h>
2121
#include <linux/tracepoint-defs.h>
22+
#include <linux/static_call.h>
2223

2324
struct module;
2425
struct tracepoint;
@@ -92,7 +93,9 @@ extern int syscall_regfunc(void);
9293
extern void syscall_unregfunc(void);
9394
#endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
9495

96+
#ifndef PARAMS
9597
#define PARAMS(args...) args
98+
#endif
9699

97100
#define TRACE_DEFINE_ENUM(x)
98101
#define TRACE_DEFINE_SIZEOF(x)
@@ -148,6 +151,12 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
148151

149152
#ifdef TRACEPOINTS_ENABLED
150153

154+
#ifdef CONFIG_HAVE_STATIC_CALL
155+
#define __DO_TRACE_CALL(name) static_call(tp_func_##name)
156+
#else
157+
#define __DO_TRACE_CALL(name) __tracepoint_iter_##name
158+
#endif /* CONFIG_HAVE_STATIC_CALL */
159+
151160
/*
152161
* it_func[0] is never NULL because there is at least one element in the array
153162
* when the array itself is non NULL.
@@ -157,12 +166,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
157166
* has a "void" prototype, then it is invalid to declare a function
158167
* as "(void *, void)".
159168
*/
160-
#define __DO_TRACE(tp, proto, args, cond, rcuidle) \
169+
#define __DO_TRACE(name, proto, args, cond, rcuidle) \
161170
do { \
162171
struct tracepoint_func *it_func_ptr; \
163-
void *it_func; \
164-
void *__data; \
165172
int __maybe_unused __idx = 0; \
173+
void *__data; \
166174
\
167175
if (!(cond)) \
168176
return; \
@@ -182,14 +190,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
182190
rcu_irq_enter_irqson(); \
183191
} \
184192
\
185-
it_func_ptr = rcu_dereference_raw((tp)->funcs); \
186-
\
193+
it_func_ptr = \
194+
rcu_dereference_raw((&__tracepoint_##name)->funcs); \
187195
if (it_func_ptr) { \
188-
do { \
189-
it_func = (it_func_ptr)->func; \
190-
__data = (it_func_ptr)->data; \
191-
((void(*)(proto))(it_func))(args); \
192-
} while ((++it_func_ptr)->func); \
196+
__data = (it_func_ptr)->data; \
197+
__DO_TRACE_CALL(name)(args); \
193198
} \
194199
\
195200
if (rcuidle) { \
@@ -205,7 +210,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
205210
static inline void trace_##name##_rcuidle(proto) \
206211
{ \
207212
if (static_key_false(&__tracepoint_##name.key)) \
208-
__DO_TRACE(&__tracepoint_##name, \
213+
__DO_TRACE(name, \
209214
TP_PROTO(data_proto), \
210215
TP_ARGS(data_args), \
211216
TP_CONDITION(cond), 1); \
@@ -227,11 +232,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
227232
* poking RCU a bit.
228233
*/
229234
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
235+
extern int __tracepoint_iter_##name(data_proto); \
236+
DECLARE_STATIC_CALL(tp_func_##name, __tracepoint_iter_##name); \
230237
extern struct tracepoint __tracepoint_##name; \
231238
static inline void trace_##name(proto) \
232239
{ \
233240
if (static_key_false(&__tracepoint_##name.key)) \
234-
__DO_TRACE(&__tracepoint_##name, \
241+
__DO_TRACE(name, \
235242
TP_PROTO(data_proto), \
236243
TP_ARGS(data_args), \
237244
TP_CONDITION(cond), 0); \
@@ -277,21 +284,50 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
277284
* structures, so we create an array of pointers that will be used for iteration
278285
* on the tracepoints.
279286
*/
280-
#define DEFINE_TRACE_FN(name, reg, unreg) \
281-
static const char __tpstrtab_##name[] \
282-
__section(__tracepoints_strings) = #name; \
283-
struct tracepoint __tracepoint_##name __used \
284-
__section(__tracepoints) = \
285-
{ __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
286-
__TRACEPOINT_ENTRY(name);
287+
#define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \
288+
static const char __tpstrtab_##_name[] \
289+
__section(__tracepoints_strings) = #_name; \
290+
extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \
291+
int __tracepoint_iter_##_name(void *__data, proto); \
292+
struct tracepoint __tracepoint_##_name __used \
293+
__section(__tracepoints) = { \
294+
.name = __tpstrtab_##_name, \
295+
.key = STATIC_KEY_INIT_FALSE, \
296+
.static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \
297+
.static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \
298+
.iterator = &__tracepoint_iter_##_name, \
299+
.regfunc = _reg, \
300+
.unregfunc = _unreg, \
301+
.funcs = NULL }; \
302+
__TRACEPOINT_ENTRY(_name); \
303+
int __tracepoint_iter_##_name(void *__data, proto) \
304+
{ \
305+
struct tracepoint_func *it_func_ptr; \
306+
void *it_func; \
307+
\
308+
it_func_ptr = \
309+
rcu_dereference_raw((&__tracepoint_##_name)->funcs); \
310+
do { \
311+
it_func = (it_func_ptr)->func; \
312+
__data = (it_func_ptr)->data; \
313+
((void(*)(void *, proto))(it_func))(__data, args); \
314+
} while ((++it_func_ptr)->func); \
315+
return 0; \
316+
} \
317+
DEFINE_STATIC_CALL(tp_func_##_name, __tracepoint_iter_##_name);
287318

288-
#define DEFINE_TRACE(name) \
289-
DEFINE_TRACE_FN(name, NULL, NULL);
319+
#define DEFINE_TRACE(name, proto, args) \
320+
DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
290321

291322
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
292-
EXPORT_SYMBOL_GPL(__tracepoint_##name)
323+
EXPORT_SYMBOL_GPL(__tracepoint_##name); \
324+
EXPORT_SYMBOL_GPL(__tracepoint_iter_##name); \
325+
EXPORT_STATIC_CALL_GPL(tp_func_##name)
293326
#define EXPORT_TRACEPOINT_SYMBOL(name) \
294-
EXPORT_SYMBOL(__tracepoint_##name)
327+
EXPORT_SYMBOL(__tracepoint_##name); \
328+
EXPORT_SYMBOL(__tracepoint_iter_##name); \
329+
EXPORT_STATIC_CALL(tp_func_##name)
330+
295331

296332
#else /* !TRACEPOINTS_ENABLED */
297333
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
@@ -320,8 +356,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p)
320356
return false; \
321357
}
322358

323-
#define DEFINE_TRACE_FN(name, reg, unreg)
324-
#define DEFINE_TRACE(name)
359+
#define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
360+
#define DEFINE_TRACE(name, proto, args)
325361
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
326362
#define EXPORT_TRACEPOINT_SYMBOL(name)
327363

‎include/trace/define_trace.h‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
#undef TRACE_EVENT
2727
#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
28-
DEFINE_TRACE(name)
28+
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
2929

3030
#undef TRACE_EVENT_CONDITION
3131
#define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
@@ -39,12 +39,12 @@
3939
#undef TRACE_EVENT_FN
4040
#define TRACE_EVENT_FN(name, proto, args, tstruct, \
4141
assign, print, reg, unreg) \
42-
DEFINE_TRACE_FN(name, reg, unreg)
42+
DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
4343

4444
#undef TRACE_EVENT_FN_COND
4545
#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
4646
assign, print, reg, unreg) \
47-
DEFINE_TRACE_FN(name, reg, unreg)
47+
DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
4848

4949
#undef TRACE_EVENT_NOP
5050
#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print)
@@ -54,23 +54,23 @@
5454

5555
#undef DEFINE_EVENT
5656
#define DEFINE_EVENT(template, name, proto, args) \
57-
DEFINE_TRACE(name)
57+
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
5858

5959
#undef DEFINE_EVENT_FN
6060
#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
61-
DEFINE_TRACE_FN(name, reg, unreg)
61+
DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
6262

6363
#undef DEFINE_EVENT_PRINT
6464
#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
65-
DEFINE_TRACE(name)
65+
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
6666

6767
#undef DEFINE_EVENT_CONDITION
6868
#define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
6969
DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
7070

7171
#undef DECLARE_TRACE
7272
#define DECLARE_TRACE(name, proto, args) \
73-
DEFINE_TRACE(name)
73+
DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
7474

7575
#undef TRACE_INCLUDE
7676
#undef __TRACE_INCLUDE

‎kernel/tracepoint.c‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,20 @@ static void *func_remove(struct tracepoint_func **funcs,
221221
return old;
222222
}
223223

224+
static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
225+
{
226+
void *func = tp->iterator;
227+
228+
/* Synthetic events do not have static call sites */
229+
if (!tp->static_call_key)
230+
return;
231+
232+
if (!tp_funcs[1].func)
233+
func = tp_funcs[0].func;
234+
235+
__static_call_update(tp->static_call_key, tp->static_call_tramp, func);
236+
}
237+
224238
/*
225239
* Add the probe function to a tracepoint.
226240
*/
@@ -251,8 +265,9 @@ static int tracepoint_add_func(struct tracepoint *tp,
251265
* include/linux/tracepoint.h using rcu_dereference_sched().
252266
*/
253267
rcu_assign_pointer(tp->funcs, tp_funcs);
254-
if (!static_key_enabled(&tp->key))
255-
static_key_slow_inc(&tp->key);
268+
tracepoint_update_call(tp, tp_funcs);
269+
static_key_enable(&tp->key);
270+
256271
release_probes(old);
257272
return 0;
258273
}
@@ -281,9 +296,11 @@ static int tracepoint_remove_func(struct tracepoint *tp,
281296
if (tp->unregfunc && static_key_enabled(&tp->key))
282297
tp->unregfunc();
283298

284-
if (static_key_enabled(&tp->key))
285-
static_key_slow_dec(&tp->key);
299+
static_key_disable(&tp->key);
300+
} else {
301+
tracepoint_update_call(tp, tp_funcs);
286302
}
303+
287304
rcu_assign_pointer(tp->funcs, tp_funcs);
288305
release_probes(old);
289306
return 0;

0 commit comments

Comments
 (0)