Skip to content

Commit 6832d96

Browse files
torvaldstorvalds
committed
Merge branch 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timers/nohz changes from Ingo Molnar: "It mostly contains fixes and full dynticks off-case optimizations, by Frederic Weisbecker" * 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (24 commits) nohz: Include local CPU in full dynticks global kick nohz: Optimize full dynticks's sched hooks with static keys nohz: Optimize full dynticks state checks with static keys nohz: Rename a few state variables vtime: Always debug check snapshot source _before_ updating it vtime: Always scale generic vtime accounting results vtime: Optimize full dynticks accounting off case with static keys vtime: Describe overriden functions in dedicated arch headers m68k: hardirq_count() only need preempt_mask.h hardirq: Split preempt count mask definitions context_tracking: Split low level state headers vtime: Fix racy cputime delta update vtime: Remove a few unneeded generic vtime state checks context_tracking: User/kernel broundary cross trace events context_tracking: Optimize context switch off case with static keys context_tracking: Optimize guest APIs off case with static key context_tracking: Optimize main APIs off case with static key context_tracking: Ground setup for static key use context_tracking: Remove full dynticks' hacky dependency on wide context tracking nohz: Only enable context tracking on full dynticks CPUs ...
2 parents 228abe7 + c2e7fcf commit 6832d96

File tree

21 files changed

+545
-327
lines changed

21 files changed

+545
-327
lines changed

‎arch/ia64/include/asm/Kbuild‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ generic-y += clkdev.h
33
generic-y += exec.h
44
generic-y += kvm_para.h
55
generic-y += trace_clock.h
6+
generic-y += vtime.h

‎arch/m68k/include/asm/irqflags.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
#include <linux/types.h>
55
#ifdef CONFIG_MMU
6-
#include <linux/hardirq.h>
6+
#include <linux/preempt_mask.h>
77
#endif
88
#include <linux/preempt.h>
99
#include <asm/thread_info.h>

‎arch/powerpc/include/asm/Kbuild‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
generic-y += clkdev.h
33
generic-y += rwsem.h
44
generic-y += trace_clock.h
5+
generic-y += vtime.h

‎arch/s390/include/asm/cputime.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@
1313
#include <asm/div64.h>
1414

1515

16-
#define __ARCH_HAS_VTIME_ACCOUNT
17-
#define __ARCH_HAS_VTIME_TASK_SWITCH
18-
1916
/* We want to use full resolution of the CPU timer: 2**-12 micro-seconds. */
2017

2118
typedef unsigned long long __nocast cputime_t;

‎arch/s390/include/asm/vtime.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#ifndef _S390_VTIME_H
2+
#define _S390_VTIME_H
3+
4+
#define __ARCH_HAS_VTIME_ACCOUNT
5+
#define __ARCH_HAS_VTIME_TASK_SWITCH
6+
7+
#endif /* _S390_VTIME_H */

‎arch/s390/kernel/vtime.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include <asm/irq_regs.h>
2020
#include <asm/cputime.h>
2121
#include <asm/vtimer.h>
22+
#include <asm/vtime.h>
2223
#include <asm/irq.h>
2324
#include "entry.h"
2425

‎include/asm-generic/vtime.h‎

Whitespace-only changes.

‎include/linux/context_tracking.h‎

Lines changed: 69 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -2,100 +2,110 @@
22
#define _LINUX_CONTEXT_TRACKING_H
33

44
#include <linux/sched.h>
5-
#include <linux/percpu.h>
65
#include <linux/vtime.h>
6+
#include <linux/context_tracking_state.h>
77
#include <asm/ptrace.h>
88

9-
struct context_tracking {
10-
/*
11-
* When active is false, probes are unset in order
12-
* to minimize overhead: TIF flags are cleared
13-
* and calls to user_enter/exit are ignored. This
14-
* may be further optimized using static keys.
15-
*/
16-
bool active;
17-
enum ctx_state {
18-
IN_KERNEL = 0,
19-
IN_USER,
20-
} state;
21-
};
22-
23-
static inline void __guest_enter(void)
24-
{
25-
/*
26-
* This is running in ioctl context so we can avoid
27-
* the call to vtime_account() with its unnecessary idle check.
28-
*/
29-
vtime_account_system(current);
30-
current->flags |= PF_VCPU;
31-
}
32-
33-
static inline void __guest_exit(void)
34-
{
35-
/*
36-
* This is running in ioctl context so we can avoid
37-
* the call to vtime_account() with its unnecessary idle check.
38-
*/
39-
vtime_account_system(current);
40-
current->flags &= ~PF_VCPU;
41-
}
429

4310
#ifdef CONFIG_CONTEXT_TRACKING
44-
DECLARE_PER_CPU(struct context_tracking, context_tracking);
11+
extern void context_tracking_cpu_set(int cpu);
4512

46-
static inline bool context_tracking_in_user(void)
13+
extern void context_tracking_user_enter(void);
14+
extern void context_tracking_user_exit(void);
15+
extern void __context_tracking_task_switch(struct task_struct *prev,
16+
struct task_struct *next);
17+
18+
static inline void user_enter(void)
4719
{
48-
return __this_cpu_read(context_tracking.state) == IN_USER;
49-
}
20+
if (static_key_false(&context_tracking_enabled))
21+
context_tracking_user_enter();
5022

51-
static inline bool context_tracking_active(void)
23+
}
24+
static inline void user_exit(void)
5225
{
53-
return __this_cpu_read(context_tracking.active);
26+
if (static_key_false(&context_tracking_enabled))
27+
context_tracking_user_exit();
5428
}
5529

56-
extern void user_enter(void);
57-
extern void user_exit(void);
58-
59-
extern void guest_enter(void);
60-
extern void guest_exit(void);
61-
6230
static inline enum ctx_state exception_enter(void)
6331
{
6432
enum ctx_state prev_ctx;
6533

34+
if (!static_key_false(&context_tracking_enabled))
35+
return 0;
36+
6637
prev_ctx = this_cpu_read(context_tracking.state);
67-
user_exit();
38+
context_tracking_user_exit();
6839

6940
return prev_ctx;
7041
}
7142

7243
static inline void exception_exit(enum ctx_state prev_ctx)
7344
{
74-
if (prev_ctx == IN_USER)
75-
user_enter();
45+
if (static_key_false(&context_tracking_enabled)) {
46+
if (prev_ctx == IN_USER)
47+
context_tracking_user_enter();
48+
}
7649
}
7750

78-
extern void context_tracking_task_switch(struct task_struct *prev,
79-
struct task_struct *next);
51+
static inline void context_tracking_task_switch(struct task_struct *prev,
52+
struct task_struct *next)
53+
{
54+
if (static_key_false(&context_tracking_enabled))
55+
__context_tracking_task_switch(prev, next);
56+
}
8057
#else
81-
static inline bool context_tracking_in_user(void) { return false; }
8258
static inline void user_enter(void) { }
8359
static inline void user_exit(void) { }
60+
static inline enum ctx_state exception_enter(void) { return 0; }
61+
static inline void exception_exit(enum ctx_state prev_ctx) { }
62+
static inline void context_tracking_task_switch(struct task_struct *prev,
63+
struct task_struct *next) { }
64+
#endif /* !CONFIG_CONTEXT_TRACKING */
65+
66+
67+
#ifdef CONFIG_CONTEXT_TRACKING_FORCE
68+
extern void context_tracking_init(void);
69+
#else
70+
static inline void context_tracking_init(void) { }
71+
#endif /* CONFIG_CONTEXT_TRACKING_FORCE */
72+
8473

74+
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
8575
static inline void guest_enter(void)
8676
{
87-
__guest_enter();
77+
if (vtime_accounting_enabled())
78+
vtime_guest_enter(current);
79+
else
80+
current->flags |= PF_VCPU;
8881
}
8982

9083
static inline void guest_exit(void)
9184
{
92-
__guest_exit();
85+
if (vtime_accounting_enabled())
86+
vtime_guest_exit(current);
87+
else
88+
current->flags &= ~PF_VCPU;
9389
}
9490

95-
static inline enum ctx_state exception_enter(void) { return 0; }
96-
static inline void exception_exit(enum ctx_state prev_ctx) { }
97-
static inline void context_tracking_task_switch(struct task_struct *prev,
98-
struct task_struct *next) { }
99-
#endif /* !CONFIG_CONTEXT_TRACKING */
91+
#else
92+
static inline void guest_enter(void)
93+
{
94+
/*
95+
* This is running in ioctl context so its safe
96+
* to assume that it's the stime pending cputime
97+
* to flush.
98+
*/
99+
vtime_account_system(current);
100+
current->flags |= PF_VCPU;
101+
}
102+
103+
static inline void guest_exit(void)
104+
{
105+
/* Flush the guest cputime we spent on the guest */
106+
vtime_account_system(current);
107+
current->flags &= ~PF_VCPU;
108+
}
109+
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
100110

101111
#endif

‎include/linux/context_tracking_state.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#ifndef _LINUX_CONTEXT_TRACKING_STATE_H
2+
#define _LINUX_CONTEXT_TRACKING_STATE_H
3+
4+
#include <linux/percpu.h>
5+
#include <linux/static_key.h>
6+
7+
struct context_tracking {
8+
/*
9+
* When active is false, probes are unset in order
10+
* to minimize overhead: TIF flags are cleared
11+
* and calls to user_enter/exit are ignored. This
12+
* may be further optimized using static keys.
13+
*/
14+
bool active;
15+
enum ctx_state {
16+
IN_KERNEL = 0,
17+
IN_USER,
18+
} state;
19+
};
20+
21+
#ifdef CONFIG_CONTEXT_TRACKING
22+
extern struct static_key context_tracking_enabled;
23+
DECLARE_PER_CPU(struct context_tracking, context_tracking);
24+
25+
static inline bool context_tracking_in_user(void)
26+
{
27+
return __this_cpu_read(context_tracking.state) == IN_USER;
28+
}
29+
30+
static inline bool context_tracking_active(void)
31+
{
32+
return __this_cpu_read(context_tracking.active);
33+
}
34+
#else
35+
static inline bool context_tracking_in_user(void) { return false; }
36+
static inline bool context_tracking_active(void) { return false; }
37+
#endif /* CONFIG_CONTEXT_TRACKING */
38+
39+
#endif

‎include/linux/hardirq.h‎

Lines changed: 1 addition & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,126 +1,11 @@
11
#ifndef LINUX_HARDIRQ_H
22
#define LINUX_HARDIRQ_H
33

4-
#include <linux/preempt.h>
4+
#include <linux/preempt_mask.h>
55
#include <linux/lockdep.h>
66
#include <linux/ftrace_irq.h>
77
#include <linux/vtime.h>
8-
#include <asm/hardirq.h>
98

10-
/*
11-
* We put the hardirq and softirq counter into the preemption
12-
* counter. The bitmask has the following meaning:
13-
*
14-
* - bits 0-7 are the preemption count (max preemption depth: 256)
15-
* - bits 8-15 are the softirq count (max # of softirqs: 256)
16-
*
17-
* The hardirq count can in theory reach the same as NR_IRQS.
18-
* In reality, the number of nested IRQS is limited to the stack
19-
* size as well. For archs with over 1000 IRQS it is not practical
20-
* to expect that they will all nest. We give a max of 10 bits for
21-
* hardirq nesting. An arch may choose to give less than 10 bits.
22-
* m68k expects it to be 8.
23-
*
24-
* - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024)
25-
* - bit 26 is the NMI_MASK
26-
* - bit 27 is the PREEMPT_ACTIVE flag
27-
*
28-
* PREEMPT_MASK: 0x000000ff
29-
* SOFTIRQ_MASK: 0x0000ff00
30-
* HARDIRQ_MASK: 0x03ff0000
31-
* NMI_MASK: 0x04000000
32-
*/
33-
#define PREEMPT_BITS 8
34-
#define SOFTIRQ_BITS 8
35-
#define NMI_BITS 1
36-
37-
#define MAX_HARDIRQ_BITS 10
38-
39-
#ifndef HARDIRQ_BITS
40-
# define HARDIRQ_BITS MAX_HARDIRQ_BITS
41-
#endif
42-
43-
#if HARDIRQ_BITS > MAX_HARDIRQ_BITS
44-
#error HARDIRQ_BITS too high!
45-
#endif
46-
47-
#define PREEMPT_SHIFT 0
48-
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
49-
#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
50-
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
51-
52-
#define __IRQ_MASK(x) ((1UL << (x))-1)
53-
54-
#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
55-
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
56-
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
57-
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
58-
59-
#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
60-
#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
61-
#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
62-
#define NMI_OFFSET (1UL << NMI_SHIFT)
63-
64-
#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)
65-
66-
#ifndef PREEMPT_ACTIVE
67-
#define PREEMPT_ACTIVE_BITS 1
68-
#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)
69-
#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
70-
#endif
71-
72-
#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT + NMI_BITS))
73-
#error PREEMPT_ACTIVE is too low!
74-
#endif
75-
76-
#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
77-
#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
78-
#define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
79-
| NMI_MASK))
80-
81-
/*
82-
* Are we doing bottom half or hardware interrupt processing?
83-
* Are we in a softirq context? Interrupt context?
84-
* in_softirq - Are we currently processing softirq or have bh disabled?
85-
* in_serving_softirq - Are we currently processing softirq?
86-
*/
87-
#define in_irq() (hardirq_count())
88-
#define in_softirq() (softirq_count())
89-
#define in_interrupt() (irq_count())
90-
#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)
91-
92-
/*
93-
* Are we in NMI context?
94-
*/
95-
#define in_nmi() (preempt_count() & NMI_MASK)
96-
97-
#if defined(CONFIG_PREEMPT_COUNT)
98-
# define PREEMPT_CHECK_OFFSET 1
99-
#else
100-
# define PREEMPT_CHECK_OFFSET 0
101-
#endif
102-
103-
/*
104-
* Are we running in atomic context? WARNING: this macro cannot
105-
* always detect atomic context; in particular, it cannot know about
106-
* held spinlocks in non-preemptible kernels. Thus it should not be
107-
* used in the general case to determine whether sleeping is possible.
108-
* Do not use in_atomic() in driver code.
109-
*/
110-
#define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != 0)
111-
112-
/*
113-
* Check whether we were atomic before we did preempt_disable():
114-
* (used by the scheduler, *after* releasing the kernel lock)
115-
*/
116-
#define in_atomic_preempt_off() \
117-
((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
118-
119-
#ifdef CONFIG_PREEMPT_COUNT
120-
# define preemptible() (preempt_count() == 0 && !irqs_disabled())
121-
#else
122-
# define preemptible() 0
123-
#endif
1249

12510
#if defined(CONFIG_SMP) || defined(CONFIG_GENERIC_HARDIRQS)
12611
extern void synchronize_irq(unsigned int irq);

0 commit comments

Comments
 (0)