percpu: add PERCPU variables handling macros

Meta handlig macros: * PERCPU_VAR(var) - returns percpu 'var' variable * PERCPU_OFFSET(var) - returns offset to 'var' variable * PERCPU_TYPE(var) - returns type of the 'var' variable * PERCPU_GET(var) - returns value of 'var' variable * PERCPU_SET(var) - sets value of scalar 'var' variable types Signed-off-by: Pawel Wieczorkiewicz <wipawel@amazon.de>
KernelTestFramework · Nov 9, 2020 · b7e9594 · b7e9594
1 parent 0546d88
commit b7e9594
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 7 deletions.
diff --git a/arch/x86/traps.c b/arch/x86/traps.c
@@ -43,9 +43,9 @@ extern void asm_interrupt_handler_pit(void);
 
 static void ret2kern_handler(void) {
     /* clang-format off */
-    asm volatile("mov %%gs:(%0), %%" STR(_ASM_SP) "\n"
+    asm volatile("mov %%gs:%[sp], %%" STR(_ASM_SP) "\n"
                  "POPF \n"
-                 ::"r"(offsetof(percpu_t, ret2kern_sp)));
+                 :: [ sp ] "m"(ACCESS_ONCE(PERCPU_VAR(ret2kern_sp))));
     /* clang-format on */
 }
 

diff --git a/common/kernel.c b/common/kernel.c
@@ -36,8 +36,8 @@ extern int usermode_call_asm(user_func_t fn, void *fn_arg, unsigned long ret2ker
                              unsigned long user_stack);
 
 int usermode_call(user_func_t fn, void *fn_arg) {
-    return usermode_call_asm(fn, fn_arg, offsetof(percpu_t, ret2kern_sp),
-                             offsetof(percpu_t, user_stack));
+    return usermode_call_asm(fn, fn_arg, PERCPU_OFFSET(ret2kern_sp),
+                             PERCPU_OFFSET(user_stack));
 }
 
 void kernel_main(void) {

diff --git a/include/compiler.h b/include/compiler.h
@@ -49,6 +49,7 @@
 #define GHZ(x) (MHZ(x) * 1000)
 
 #ifndef __ASSEMBLY__
+typedef uint64_t off_t;
 #define _ptr(val) ((void *) (unsigned long) (val))
 #define _ul(val)  ((unsigned long) (val))
 #define _int(val) ((int) (val))
@@ -91,8 +92,9 @@
 #define likely(x)   __builtin_expect(!!(x), 1)
 #define unlikely(x) __builtin_expect(!!(x), 0)
 
+#define memberof(type, member) (((type *) 0)->member)
 #ifndef offsetof
-#define offsetof(type, member) ((off_t) & ((type *) 0)->member)
+#define offsetof(type, member) ((off_t) &memberof((type), member))
 #endif
 
 #define BUILD_BUG_ON(cond) ({ _Static_assert(!(cond), "!(" STR(cond) ")"); })
@@ -104,7 +106,7 @@
 #define container_of(ptr, type, member)                                                  \
     ({                                                                                   \
         void *_ptr = (void *) (ptr);                                                     \
-        BUILD_BUG_ON(!__same_type(*(ptr), ((type *) 0)->member) &&                       \
+        BUILD_BUG_ON(!__same_type(*(ptr), memberof(type, member)) &&                     \
                      !__same_type(*(ptr), void));                                        \
         ((type *) (_ptr - offsetof(type, member)));                                      \
     })

diff --git a/include/percpu.h b/include/percpu.h
@@ -57,6 +57,68 @@ struct percpu {
 } __aligned(PAGE_SIZE);
 typedef struct percpu percpu_t;
 
+#define PERCPU_VAR(variable)    memberof(percpu_t, variable)
+#define PERCPU_OFFSET(variable) ((off_t) &PERCPU_VAR(variable))
+#define PERCPU_TYPE(variable)   typeof(PERCPU_VAR(variable))
+
+#define PERCPU_GET(variable)                                                             \
+    ({                                                                                   \
+        PERCPU_TYPE(variable) __local_var;                                               \
+        asm volatile("mov %%gs:%[percpu_var], %[local_var]"                              \
+                     : [ local_var ] "=r"(__local_var)                                   \
+                     : [ percpu_var ] "m"(ACCESS_ONCE(PERCPU_VAR(variable))));           \
+        __local_var;                                                                     \
+    })
+
+#define PERCPU_SET_BYTE(variable, value)                                                 \
+    ({                                                                                   \
+        asm volatile("movb %[val], %%gs:%[percpu_var]"                                   \
+                     : [ percpu_var ] "=m"(ACCESS_ONCE(PERCPU_VAR(variable)))            \
+                     : [ val ] "r"((uint8_t) value));                                    \
+    })
+
+#define PERCPU_SET_WORD(variable, value)                                                 \
+    ({                                                                                   \
+        asm volatile("movw %[val], %%gs:%[percpu_var]"                                   \
+                     : [ percpu_var ] "=m"(ACCESS_ONCE(PERCPU_VAR(variable)))            \
+                     : [ val ] "r"((uint16_t) value));                                   \
+    })
+
+#define PERCPU_SET_DWORD(variable, value)                                                \
+    ({                                                                                   \
+        asm volatile("movl %[val], %%gs:%[percpu_var]"                                   \
+                     : [ percpu_var ] "=m"(ACCESS_ONCE(PERCPU_VAR(variable)))            \
+                     : [ val ] "r"((uint32_t) value));                                   \
+    })
+
+#if defined(__x86_64__)
+#define PERCPU_SET_QWORD(variable, value)                                                \
+    ({                                                                                   \
+        asm volatile("movq %[val], %%gs:%[percpu_var]"                                   \
+                     : [ percpu_var ] "=m"(ACCESS_ONCE(PERCPU_VAR(variable)))            \
+                     : [ val ] "r"((uint64_t) value));                                   \
+    })
+#endif
+
+#if defined(__x86_64__)
+#define PERCPU_SET_MAX_SIZE sizeof(uint64_t)
+#else
+#define PERCPU_SET_MAX_SIZE sizeof(uint32_t)
+#endif
+
+/* clang-format off */
+#define PERCPU_SET(variable, value)                                                      \
+    ({                                                                                   \
+        BUILD_BUG_ON(sizeof(PERCPU_TYPE(variable)) > PERCPU_SET_MAX_SIZE);               \
+        size_t __var_size = sizeof(PERCPU_TYPE(variable));                               \
+        if (__var_size == sizeof(uint8_t))       PERCPU_SET_BYTE(variable, value);       \
+        else if (__var_size == sizeof(uint16_t)) PERCPU_SET_WORD(variable, value);       \
+        else if (__var_size == sizeof(uint32_t)) PERCPU_SET_DWORD(variable, value);      \
+        else if (__var_size == sizeof(uint64_t)) PERCPU_SET_QWORD(variable, value);      \
+        else BUG();                                                                      \
+    })
+/* clang-format on */
+
 /* External declarations */
 
 extern void init_percpu(void);

diff --git a/smp/smp.c b/smp/smp.c
@@ -66,7 +66,7 @@ static __text_init void boot_cpu(unsigned int cpu) {
     percpu_t *percpu = get_percpu_page(cpu);
     uint64_t icr;
 
-    if (percpu->bsp)
+    if (PERCPU_GET(bsp))
         return;
 
     ap_cpuid = cpu;