From ffbe4d995a60d5d404ed8dc8ac674a28f9923e87 Mon Sep 17 00:00:00 2001 From: Adam Borowski Date: Thu, 24 Jun 2021 18:08:42 +0000 Subject: [PATCH] pmem2: arm64: fix data loss on ARMv8.2+ (improper flushing) ARMv8.2 separates flush-for-coherency from flush-for-persistency as two instructions: DC CVAC and DC CVAP. It's the latter that is of interest to us, and it is up to the processor to know where the Point of Persistency happens to be. The documentation is very clear, and this PIRL talk dispels all doubt: https://www.youtube.com/watch?v=8QAuN8CL5Zg On the other hand, I found no clear answer whether DC CVAC might be enough on some pre-8.2 hardware. There's no other option though (CVAC is the heaviest flush available). This question is probably moot as pre-8.2 CPUs, while ubiquitous in dev boards, are gone in datacenter gear. --- src/libpmem2/aarch64/arm_cacheops.h | 13 ++++++-- src/libpmem2/aarch64/flush.h | 22 +++++++++++-- src/libpmem2/aarch64/init.c | 50 +++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/src/libpmem2/aarch64/arm_cacheops.h b/src/libpmem2/aarch64/arm_cacheops.h index 5eab9995d63..b78f6f21d2c 100644 --- a/src/libpmem2/aarch64/arm_cacheops.h +++ b/src/libpmem2/aarch64/arm_cacheops.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2014-2020, Intel Corporation */ +/* Copyright 2014-2021, Intel Corporation */ /* * ARM inline assembly to flush and invalidate caches * clwb => dc cvac @@ -11,8 +11,9 @@ /* * Cache instructions on ARM: * ARMv8.0-a DC CVAC - cache clean to Point of Coherency - * Meant for thread synchronization, usually implies - * real memory flush but may mean less. + * Meant for thread synchronization, before 8.2 may or + * may not flush memory, on 8.2+ the following is + * needed: * ARMv8.2-a DC CVAP - cache clean to Point of Persistency * Meant exactly for our use. * ARMv8.5-a DC CVADP - cache clean to Point of Deep Persistency @@ -54,6 +55,12 @@ arm_clean_va_to_poc(void const *p __attribute__((unused))) asm volatile("dc cvac, %0" : : "r" (p) : "memory"); } +static inline void +arm_clean_va_to_pop(void const *p __attribute__((unused))) +{ + asm volatile(".arch armv8.2-a\n dc cvap, %0" : : "r" (p) : "memory"); +} + static inline void arm_store_memory_barrier(void) { diff --git a/src/libpmem2/aarch64/flush.h b/src/libpmem2/aarch64/flush.h index 880d15b7df0..4296a5df237 100644 --- a/src/libpmem2/aarch64/flush.h +++ b/src/libpmem2/aarch64/flush.h @@ -11,10 +11,10 @@ #define FLUSH_ALIGN ((uintptr_t)64) /* - * flush_dcache_nolog -- flush the CPU cache, using DC CVAC + * flush_poc_nolog -- flush the CPU cache, using DC CVAC */ static force_inline void -flush_dcache_nolog(const void *addr, size_t len) +flush_poc_nolog(const void *addr, size_t len) { uintptr_t uptr; @@ -28,4 +28,22 @@ flush_dcache_nolog(const void *addr, size_t len) } } +/* + * flush_pop_nolog -- flush the CPU cache, using DC CVAP + */ +static force_inline void +flush_pop_nolog(const void *addr, size_t len) +{ + uintptr_t uptr; + + /* + * Loop through cache-line-size (typically 64B) aligned chunks + * covering the given range. + */ + for (uptr = (uintptr_t)addr & ~(FLUSH_ALIGN - 1); + uptr < (uintptr_t)addr + len; uptr += FLUSH_ALIGN) { + arm_clean_va_to_pop((char *)uptr); + } +} + #endif diff --git a/src/libpmem2/aarch64/init.c b/src/libpmem2/aarch64/init.c index fe5b8c4acd4..b38956429c8 100644 --- a/src/libpmem2/aarch64/init.c +++ b/src/libpmem2/aarch64/init.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2014-2019, Intel Corporation */ +/* Copyright 2014-2021, Intel Corporation */ #include +#include +#include #include "auto_flush.h" #include "flush.h" @@ -19,14 +21,45 @@ memory_barrier(void) } /* - * flush_dcache -- (internal) flush the CPU cache + * flush_poc -- (internal) flush the dcache to Point of Coherency, + * available on all ARMv8+. It does _not_ flush to dimms + * on new CPUs, and is ill-specified earlier. */ static void -flush_dcache(const void *addr, size_t len) +flush_poc(const void *addr, size_t len) { LOG(15, "addr %p len %zu", addr, len); - flush_dcache_nolog(addr, len); + flush_poc_nolog(addr, len); +} + +/* + * flush_pop -- (internal) flush the dcache to Point of Persistency, + * available on ARMv8.2+ + */ +static void +flush_pop(const void *addr, size_t len) +{ + LOG(15, "addr %p len %zu", addr, len); + + flush_pop_nolog(addr, len); +} + +/* + * is_dcpop_available -- (internal) determine dcpop cpuid flag using hwcaps + */ +static int +is_dc_pop_available(void) +{ + LOG(15, NULL); + + /* + * Shouldn't ever fail, but if it does, error is reported as -1 + * which conveniently includes all bits. We then assume PoP flushes + * are required -- safer on any hardware suspected of actually being + * capable of pmem, cleanly crashing with SIGILL on old gear. + */ + return getauxval(AT_HWCAP) & HWCAP_DCPOP; } /* @@ -38,10 +71,15 @@ pmem2_arch_init(struct pmem2_arch_info *info) LOG(3, NULL); info->fence = memory_barrier; - info->flush = flush_dcache; + if (is_dc_pop_available()) + info->flush = flush_pop; + else + info->flush = flush_poc; - if (info->flush == flush_dcache) + if (info->flush == flush_poc) LOG(3, "Synchronize VA to poc for ARM"); + else if (info->flush == flush_pop) + LOG(3, "Synchronize VA to pop for ARM"); else FATAL("invalid deep flush function address"); }