diff --git a/bpf/tests/builtin_memmove.h b/bpf/tests/builtin_memmove.h new file mode 100644 index 000000000000..54acb37bad25 --- /dev/null +++ b/bpf/tests/builtin_memmove.h @@ -0,0 +1,376 @@ + test___builtin_memmove1_single(8, 1); + test___builtin_memmove1_single(8, 2); + test___builtin_memmove1_single(8, 4); + test___builtin_memmove1_single(8, 6); + test___builtin_memmove1_single(8, 8); + test___builtin_memmove1_single(8, 10); + test___builtin_memmove1_single(8, 12); + test___builtin_memmove1_single(8, 14); + test___builtin_memmove1_single(8, 16); + test___builtin_memmove1_single(8, 18); + test___builtin_memmove1_single(8, 20); + test___builtin_memmove1_single(8, 22); + test___builtin_memmove1_single(8, 24); + test___builtin_memmove1_single(8, 26); + test___builtin_memmove1_single(8, 28); + test___builtin_memmove1_single(8, 30); + test___builtin_memmove1_single(8, 32); + test___builtin_memmove1_single(8, 34); + test___builtin_memmove1_single(8, 36); + test___builtin_memmove1_single(8, 38); + test___builtin_memmove1_single(8, 40); + test___builtin_memmove1_single(8, 42); + test___builtin_memmove1_single(8, 44); + test___builtin_memmove1_single(8, 46); + test___builtin_memmove1_single(8, 48); + test___builtin_memmove1_single(8, 50); + test___builtin_memmove1_single(8, 52); + test___builtin_memmove1_single(8, 54); + test___builtin_memmove1_single(8, 56); + test___builtin_memmove1_single(8, 58); + test___builtin_memmove1_single(8, 60); + test___builtin_memmove1_single(8, 62); + test___builtin_memmove1_single(8, 64); + test___builtin_memmove1_single(8, 66); + test___builtin_memmove1_single(8, 68); + test___builtin_memmove1_single(8, 70); + test___builtin_memmove1_single(8, 72); + test___builtin_memmove1_single(8, 74); + test___builtin_memmove1_single(8, 76); + test___builtin_memmove1_single(8, 78); + test___builtin_memmove1_single(8, 80); + test___builtin_memmove1_single(8, 82); + test___builtin_memmove1_single(8, 84); + test___builtin_memmove1_single(8, 86); + test___builtin_memmove1_single(8, 88); + test___builtin_memmove1_single(8, 90); + test___builtin_memmove1_single(8, 92); + test___builtin_memmove1_single(8, 94); + test___builtin_memmove1_single(8, 96); + test___builtin_memmove1_single(16, 1); + test___builtin_memmove1_single(16, 2); + test___builtin_memmove1_single(16, 4); + test___builtin_memmove1_single(16, 6); + test___builtin_memmove1_single(16, 8); + test___builtin_memmove1_single(16, 10); + test___builtin_memmove1_single(16, 12); + test___builtin_memmove1_single(16, 14); + test___builtin_memmove1_single(16, 16); + test___builtin_memmove1_single(16, 18); + test___builtin_memmove1_single(16, 20); + test___builtin_memmove1_single(16, 22); + test___builtin_memmove1_single(16, 24); + test___builtin_memmove1_single(16, 26); + test___builtin_memmove1_single(16, 28); + test___builtin_memmove1_single(16, 30); + test___builtin_memmove1_single(16, 32); + test___builtin_memmove1_single(16, 34); + test___builtin_memmove1_single(16, 36); + test___builtin_memmove1_single(16, 38); + test___builtin_memmove1_single(16, 40); + test___builtin_memmove1_single(16, 42); + test___builtin_memmove1_single(16, 44); + test___builtin_memmove1_single(16, 46); + test___builtin_memmove1_single(16, 48); + test___builtin_memmove1_single(32, 1); + test___builtin_memmove1_single(32, 2); + test___builtin_memmove1_single(32, 4); + test___builtin_memmove1_single(32, 6); + test___builtin_memmove1_single(32, 8); + test___builtin_memmove1_single(32, 10); + test___builtin_memmove1_single(32, 12); + test___builtin_memmove1_single(32, 14); + test___builtin_memmove1_single(32, 16); + test___builtin_memmove1_single(32, 18); + test___builtin_memmove1_single(32, 20); + test___builtin_memmove1_single(32, 22); + test___builtin_memmove1_single(32, 24); + test___builtin_memmove1_single(64, 1); + test___builtin_memmove1_single(64, 2); + test___builtin_memmove1_single(64, 4); + test___builtin_memmove1_single(64, 6); + test___builtin_memmove1_single(64, 8); + test___builtin_memmove1_single(64, 10); + test___builtin_memmove1_single(64, 12); + test___builtin_memmove2_single(8, 1); + test___builtin_memmove2_single(8, 2); + test___builtin_memmove2_single(8, 4); + test___builtin_memmove2_single(8, 6); + test___builtin_memmove2_single(8, 8); + test___builtin_memmove2_single(8, 10); + test___builtin_memmove2_single(8, 12); + test___builtin_memmove2_single(8, 14); + test___builtin_memmove2_single(8, 16); + test___builtin_memmove2_single(8, 18); + test___builtin_memmove2_single(8, 20); + test___builtin_memmove2_single(8, 22); + test___builtin_memmove2_single(8, 24); + test___builtin_memmove2_single(8, 26); + test___builtin_memmove2_single(8, 28); + test___builtin_memmove2_single(8, 30); + test___builtin_memmove2_single(8, 32); + test___builtin_memmove2_single(8, 34); + test___builtin_memmove2_single(8, 36); + test___builtin_memmove2_single(8, 38); + test___builtin_memmove2_single(8, 40); + test___builtin_memmove2_single(8, 42); + test___builtin_memmove2_single(8, 44); + test___builtin_memmove2_single(8, 46); + test___builtin_memmove2_single(8, 48); + test___builtin_memmove2_single(8, 50); + test___builtin_memmove2_single(8, 52); + test___builtin_memmove2_single(8, 54); + test___builtin_memmove2_single(8, 56); + test___builtin_memmove2_single(8, 58); + test___builtin_memmove2_single(8, 60); + test___builtin_memmove2_single(8, 62); + test___builtin_memmove2_single(8, 64); + test___builtin_memmove2_single(8, 66); + test___builtin_memmove2_single(8, 68); + test___builtin_memmove2_single(8, 70); + test___builtin_memmove2_single(8, 72); + test___builtin_memmove2_single(8, 74); + test___builtin_memmove2_single(8, 76); + test___builtin_memmove2_single(8, 78); + test___builtin_memmove2_single(8, 80); + test___builtin_memmove2_single(8, 82); + test___builtin_memmove2_single(8, 84); + test___builtin_memmove2_single(8, 86); + test___builtin_memmove2_single(8, 88); + test___builtin_memmove2_single(8, 90); + test___builtin_memmove2_single(8, 92); + test___builtin_memmove2_single(8, 94); + test___builtin_memmove2_single(8, 96); + test___builtin_memmove2_single(16, 1); + test___builtin_memmove2_single(16, 2); + test___builtin_memmove2_single(16, 4); + test___builtin_memmove2_single(16, 6); + test___builtin_memmove2_single(16, 8); + test___builtin_memmove2_single(16, 10); + test___builtin_memmove2_single(16, 12); + test___builtin_memmove2_single(16, 14); + test___builtin_memmove2_single(16, 16); + test___builtin_memmove2_single(16, 18); + test___builtin_memmove2_single(16, 20); + test___builtin_memmove2_single(16, 22); + test___builtin_memmove2_single(16, 24); + test___builtin_memmove2_single(16, 26); + test___builtin_memmove2_single(16, 28); + test___builtin_memmove2_single(16, 30); + test___builtin_memmove2_single(16, 32); + test___builtin_memmove2_single(16, 34); + test___builtin_memmove2_single(16, 36); + test___builtin_memmove2_single(16, 38); + test___builtin_memmove2_single(16, 40); + test___builtin_memmove2_single(16, 42); + test___builtin_memmove2_single(16, 44); + test___builtin_memmove2_single(16, 46); + test___builtin_memmove2_single(16, 48); + test___builtin_memmove2_single(32, 1); + test___builtin_memmove2_single(32, 2); + test___builtin_memmove2_single(32, 4); + test___builtin_memmove2_single(32, 6); + test___builtin_memmove2_single(32, 8); + test___builtin_memmove2_single(32, 10); + test___builtin_memmove2_single(32, 12); + test___builtin_memmove2_single(32, 14); + test___builtin_memmove2_single(32, 16); + test___builtin_memmove2_single(32, 18); + test___builtin_memmove2_single(32, 20); + test___builtin_memmove2_single(32, 22); + test___builtin_memmove2_single(32, 24); + test___builtin_memmove2_single(64, 1); + test___builtin_memmove2_single(64, 2); + test___builtin_memmove2_single(64, 4); + test___builtin_memmove2_single(64, 6); + test___builtin_memmove2_single(64, 8); + test___builtin_memmove2_single(64, 10); + test___builtin_memmove2_single(64, 12); + test___builtin_memmove3_single(8, 1); + test___builtin_memmove3_single(8, 2); + test___builtin_memmove3_single(8, 4); + test___builtin_memmove3_single(8, 6); + test___builtin_memmove3_single(8, 8); + test___builtin_memmove3_single(8, 10); + test___builtin_memmove3_single(8, 12); + test___builtin_memmove3_single(8, 14); + test___builtin_memmove3_single(8, 16); + test___builtin_memmove3_single(8, 18); + test___builtin_memmove3_single(8, 20); + test___builtin_memmove3_single(8, 22); + test___builtin_memmove3_single(8, 24); + test___builtin_memmove3_single(8, 26); + test___builtin_memmove3_single(8, 28); + test___builtin_memmove3_single(8, 30); + test___builtin_memmove3_single(8, 32); + test___builtin_memmove3_single(8, 34); + test___builtin_memmove3_single(8, 36); + test___builtin_memmove3_single(8, 38); + test___builtin_memmove3_single(8, 40); + test___builtin_memmove3_single(8, 42); + test___builtin_memmove3_single(8, 44); + test___builtin_memmove3_single(8, 46); + test___builtin_memmove3_single(8, 48); + test___builtin_memmove3_single(8, 50); + test___builtin_memmove3_single(8, 52); + test___builtin_memmove3_single(8, 54); + test___builtin_memmove3_single(8, 56); + test___builtin_memmove3_single(8, 58); + test___builtin_memmove3_single(8, 60); + test___builtin_memmove3_single(8, 62); + test___builtin_memmove3_single(8, 64); + test___builtin_memmove3_single(8, 66); + test___builtin_memmove3_single(8, 68); + test___builtin_memmove3_single(8, 70); + test___builtin_memmove3_single(8, 72); + test___builtin_memmove3_single(8, 74); + test___builtin_memmove3_single(8, 76); + test___builtin_memmove3_single(8, 78); + test___builtin_memmove3_single(8, 80); + test___builtin_memmove3_single(8, 82); + test___builtin_memmove3_single(8, 84); + test___builtin_memmove3_single(8, 86); + test___builtin_memmove3_single(8, 88); + test___builtin_memmove3_single(8, 90); + test___builtin_memmove3_single(8, 92); + test___builtin_memmove3_single(8, 94); + test___builtin_memmove3_single(8, 96); + test___builtin_memmove3_single(16, 1); + test___builtin_memmove3_single(16, 2); + test___builtin_memmove3_single(16, 4); + test___builtin_memmove3_single(16, 6); + test___builtin_memmove3_single(16, 8); + test___builtin_memmove3_single(16, 10); + test___builtin_memmove3_single(16, 12); + test___builtin_memmove3_single(16, 14); + test___builtin_memmove3_single(16, 16); + test___builtin_memmove3_single(16, 18); + test___builtin_memmove3_single(16, 20); + test___builtin_memmove3_single(16, 22); + test___builtin_memmove3_single(16, 24); + test___builtin_memmove3_single(16, 26); + test___builtin_memmove3_single(16, 28); + test___builtin_memmove3_single(16, 30); + test___builtin_memmove3_single(16, 32); + test___builtin_memmove3_single(16, 34); + test___builtin_memmove3_single(16, 36); + test___builtin_memmove3_single(16, 38); + test___builtin_memmove3_single(16, 40); + test___builtin_memmove3_single(16, 42); + test___builtin_memmove3_single(16, 44); + test___builtin_memmove3_single(16, 46); + test___builtin_memmove3_single(16, 48); + test___builtin_memmove3_single(32, 1); + test___builtin_memmove3_single(32, 2); + test___builtin_memmove3_single(32, 4); + test___builtin_memmove3_single(32, 6); + test___builtin_memmove3_single(32, 8); + test___builtin_memmove3_single(32, 10); + test___builtin_memmove3_single(32, 12); + test___builtin_memmove3_single(32, 14); + test___builtin_memmove3_single(32, 16); + test___builtin_memmove3_single(32, 18); + test___builtin_memmove3_single(32, 20); + test___builtin_memmove3_single(32, 22); + test___builtin_memmove3_single(32, 24); + test___builtin_memmove3_single(64, 1); + test___builtin_memmove3_single(64, 2); + test___builtin_memmove3_single(64, 4); + test___builtin_memmove3_single(64, 6); + test___builtin_memmove3_single(64, 8); + test___builtin_memmove3_single(64, 10); + test___builtin_memmove3_single(64, 12); + test___builtin_memmove4_single(8, 1); + test___builtin_memmove4_single(8, 2); + test___builtin_memmove4_single(8, 4); + test___builtin_memmove4_single(8, 6); + test___builtin_memmove4_single(8, 8); + test___builtin_memmove4_single(8, 10); + test___builtin_memmove4_single(8, 12); + test___builtin_memmove4_single(8, 14); + test___builtin_memmove4_single(8, 16); + test___builtin_memmove4_single(8, 18); + test___builtin_memmove4_single(8, 20); + test___builtin_memmove4_single(8, 22); + test___builtin_memmove4_single(8, 24); + test___builtin_memmove4_single(8, 26); + test___builtin_memmove4_single(8, 28); + test___builtin_memmove4_single(8, 30); + test___builtin_memmove4_single(8, 32); + test___builtin_memmove4_single(8, 34); + test___builtin_memmove4_single(8, 36); + test___builtin_memmove4_single(8, 38); + test___builtin_memmove4_single(8, 40); + test___builtin_memmove4_single(8, 42); + test___builtin_memmove4_single(8, 44); + test___builtin_memmove4_single(8, 46); + test___builtin_memmove4_single(8, 48); + test___builtin_memmove4_single(8, 50); + test___builtin_memmove4_single(8, 52); + test___builtin_memmove4_single(8, 54); + test___builtin_memmove4_single(8, 56); + test___builtin_memmove4_single(8, 58); + test___builtin_memmove4_single(8, 60); + test___builtin_memmove4_single(8, 62); + test___builtin_memmove4_single(8, 64); + test___builtin_memmove4_single(8, 66); + test___builtin_memmove4_single(8, 68); + test___builtin_memmove4_single(8, 70); + test___builtin_memmove4_single(8, 72); + test___builtin_memmove4_single(8, 74); + test___builtin_memmove4_single(8, 76); + test___builtin_memmove4_single(8, 78); + test___builtin_memmove4_single(8, 80); + test___builtin_memmove4_single(8, 82); + test___builtin_memmove4_single(8, 84); + test___builtin_memmove4_single(8, 86); + test___builtin_memmove4_single(8, 88); + test___builtin_memmove4_single(8, 90); + test___builtin_memmove4_single(8, 92); + test___builtin_memmove4_single(8, 94); + test___builtin_memmove4_single(8, 96); + test___builtin_memmove4_single(16, 1); + test___builtin_memmove4_single(16, 2); + test___builtin_memmove4_single(16, 4); + test___builtin_memmove4_single(16, 6); + test___builtin_memmove4_single(16, 8); + test___builtin_memmove4_single(16, 10); + test___builtin_memmove4_single(16, 12); + test___builtin_memmove4_single(16, 14); + test___builtin_memmove4_single(16, 16); + test___builtin_memmove4_single(16, 18); + test___builtin_memmove4_single(16, 20); + test___builtin_memmove4_single(16, 22); + test___builtin_memmove4_single(16, 24); + test___builtin_memmove4_single(16, 26); + test___builtin_memmove4_single(16, 28); + test___builtin_memmove4_single(16, 30); + test___builtin_memmove4_single(16, 32); + test___builtin_memmove4_single(16, 34); + test___builtin_memmove4_single(16, 36); + test___builtin_memmove4_single(16, 38); + test___builtin_memmove4_single(16, 40); + test___builtin_memmove4_single(16, 42); + test___builtin_memmove4_single(16, 44); + test___builtin_memmove4_single(16, 46); + test___builtin_memmove4_single(16, 48); + test___builtin_memmove4_single(32, 1); + test___builtin_memmove4_single(32, 2); + test___builtin_memmove4_single(32, 4); + test___builtin_memmove4_single(32, 6); + test___builtin_memmove4_single(32, 8); + test___builtin_memmove4_single(32, 10); + test___builtin_memmove4_single(32, 12); + test___builtin_memmove4_single(32, 14); + test___builtin_memmove4_single(32, 16); + test___builtin_memmove4_single(32, 18); + test___builtin_memmove4_single(32, 20); + test___builtin_memmove4_single(32, 22); + test___builtin_memmove4_single(32, 24); + test___builtin_memmove4_single(64, 1); + test___builtin_memmove4_single(64, 2); + test___builtin_memmove4_single(64, 4); + test___builtin_memmove4_single(64, 6); + test___builtin_memmove4_single(64, 8); + test___builtin_memmove4_single(64, 10); + test___builtin_memmove4_single(64, 12); diff --git a/bpf/tests/builtin_test.h b/bpf/tests/builtin_test.h index ad4bd53df55a..30e9707653c2 100644 --- a/bpf/tests/builtin_test.h +++ b/bpf/tests/builtin_test.h @@ -48,12 +48,22 @@ static __always_inline bool __corrupt_mem(void *d, __u32 len) return corrupted; } +static void __fill_cnt(void *buff, __u32 len) +{ + __u8 *dest = buff; + __u32 i, cnt = 0; + + for (i = 0; i < len; i++) + dest[i] = cnt++; +} + #define test___builtin_memzero_single(op, len) \ do { \ __u##op __x[len] __align_stack_8; \ __u##op __y[len] __align_stack_8; \ __bpf_memset_builtin(__y, 0, sizeof(__y)); \ __fill_rnd(__x, sizeof(__x)); \ + barrier_data(__x); \ __bpf_memzero(__x, sizeof(__x)); \ barrier_data(__x); \ barrier_data(__y); \ @@ -74,9 +84,10 @@ static void test___builtin_memzero(void) __bpf_memset_builtin(__x, 0, sizeof(__x)); \ __fill_rnd(__y, sizeof(__y)); \ __bpf_memcpy_builtin(__z, __y, sizeof(__z)); \ - __bpf_memcpy(__x, __y, sizeof(__x)); \ barrier_data(__x); \ barrier_data(__y); \ + __bpf_memcpy(__x, __y, sizeof(__x)); \ + barrier_data(__x); \ barrier_data(__z); \ assert(!__cmp_mem(__x, __z, sizeof(__x))); \ } while (0) @@ -98,8 +109,6 @@ static void test___builtin_memcpy(void) barrier_data(__x); \ barrier_data(__y); \ res = __bpf_memcmp(__x, __y, sizeof(__x)); \ - barrier_data(__x); \ - barrier_data(__y); \ assert(cor == res); \ } while (0) @@ -112,3 +121,82 @@ static void test___builtin_memcmp(void) #include "builtin_memcmp.h" } } + +/* Same as test___builtin_memcpy_single(). */ +#define test___builtin_memmove1_single(op, len) \ + do { \ + __u##op __x[len] __align_stack_8; \ + __u##op __y[len] __align_stack_8; \ + __u##op __z[len] __align_stack_8; \ + __bpf_memset_builtin(__x, 0, sizeof(__x)); \ + __fill_rnd(__y, sizeof(__y)); \ + __bpf_memcpy_builtin(__z, __y, sizeof(__z)); \ + barrier_data(__x); \ + barrier_data(__y); \ + __bpf_memmove(__x, __y, sizeof(__x)); \ + barrier_data(__x); \ + barrier_data(__z); \ + assert(!__cmp_mem(__x, __z, sizeof(__x))); \ + } while (0) + +/* Overlapping with src == dst. */ +#define test___builtin_memmove2_single(op, len) \ + do { \ + __u##op __x[len] __align_stack_8; \ + __u##op __y[len] __align_stack_8; \ + __u8 *__p_x = (__u8 *)__x; \ + __u8 *__p_y = (__u8 *)__y; \ + const __u32 off = 0; \ + __fill_cnt(__x, sizeof(__x)); \ + __bpf_memcpy_builtin(__y, __x, sizeof(__x)); \ + __bpf_memcpy_builtin(__p_y + off, __x, sizeof(__x) - off); \ + barrier_data(__x); \ + __bpf_memmove(__p_x + off, __x, sizeof(__x) - off); \ + barrier_data(__x); \ + barrier_data(__y); \ + assert(!__cmp_mem(__x, __y, sizeof(__x))); \ + } while (0) + +/* Overlapping with src < dst. */ +#define test___builtin_memmove3_single(op, len) \ + do { \ + __u##op __x[len] __align_stack_8; \ + __u##op __y[len] __align_stack_8; \ + __u8 *__p_x = (__u8 *)__x; \ + __u8 *__p_y = (__u8 *)__y; \ + const __u32 off = (sizeof(__x[0]) * len / 2) & ~1U; \ + __fill_cnt(__x, sizeof(__x)); \ + __bpf_memcpy_builtin(__y, __x, sizeof(__x)); \ + __bpf_memcpy_builtin(__p_y + off, __x, sizeof(__x) - off); \ + barrier_data(__x); \ + __bpf_memmove(__p_x + off, __x, sizeof(__x) - off); \ + barrier_data(__x); \ + barrier_data(__y); \ + assert(!__cmp_mem(__x, __y, sizeof(__x))); \ + } while (0) + +/* Overlapping with src > dst. */ +#define test___builtin_memmove4_single(op, len) \ + do { \ + __u##op __x[len] __align_stack_8; \ + __u##op __y[len] __align_stack_8; \ + __u8 *__p_x = (__u8 *)__x; \ + const __u32 off = (sizeof(__x[0]) * len / 2) & ~1U; \ + __fill_cnt(__x, sizeof(__x)); \ + __bpf_memcpy_builtin(__y, __x, sizeof(__x)); \ + __bpf_memcpy_builtin(__y, __p_x + off, sizeof(__x) - off); \ + barrier_data(__x); \ + __bpf_memmove(__x, __p_x + off, sizeof(__x) - off); \ + barrier_data(__x); \ + barrier_data(__y); \ + assert(!__cmp_mem(__x, __y, sizeof(__x))); \ + } while (0) + +static void test___builtin_memmove(void) +{ + /* ./builtin_gen memmove1 768 > builtin_memmove.h */ + /* ./builtin_gen memmove2 768 >> builtin_memmove.h */ + /* ./builtin_gen memmove3 768 >> builtin_memmove.h */ + /* ./builtin_gen memmove4 768 >> builtin_memmove.h */ + #include "builtin_memmove.h" +} diff --git a/test/bpf/unit-test.c b/test/bpf/unit-test.c index 67f9712fa0dd..3a334163f6ee 100644 --- a/test/bpf/unit-test.c +++ b/test/bpf/unit-test.c @@ -40,6 +40,7 @@ int main(int argc, char *argv[]) test___builtin_memzero(); test___builtin_memcpy(); test___builtin_memcmp(); + test___builtin_memmove(); return 0; }