Improve __copy_to_user and __copy_from_user performance

Provide a __copy_from_user that uses memcpy. On BCM2708, use
optimised memcpy/memmove/memcmp/memset implementations.

arch/arm: Add mmiocpy/set aliases for memcpy/set

See: #1082

copy_from_user: CPU_SW_DOMAIN_PAN compatibility

The downstream copy_from_user acceleration must also play nice with
CONFIG_CPU_SW_DOMAIN_PAN.

See: #1381

Signed-off-by: Phil Elwell <phil@raspberrypi.org>

Fix copy_from_user if BCM2835_FAST_MEMCPY=n

The change which introduced CONFIG_BCM2835_FAST_MEMCPY unconditionally
changed the behaviour of arm_copy_from_user. The page pinning code
is not safe on ARMv7 if LPAE & high memory is enabled and causes
crashes which look like PTE corruption.

Make __copy_from_user_memcpy conditional on CONFIG_2835_FAST_MEMCPY=y
which is really an ARMv6 / Pi1 optimization and not necessary on newer
ARM processors.

arm: fix mmap unlocks in uaccess_with_memcpy.c

This is a regression that was added with the commit 192a4e9 as of rpi-5.8.y, since that is when the move to the mmap locking API was introduced - d8ed45c

The issue is that when the patch to improve performance for the __copy_to_user and __copy_from_user functions were added for the Raspberry Pi, some of the mmaps were incorrectly mapped to write instead of read. This would cause a verity of issues, and in my case, prevent the booting of a squashfs filesystem on rpi-5.8-y and above. An example of the panic you would see from this can be seen at https://pastebin.com/raw/jBz5xCzL

Signed-off-by: Christian Lamparter <chunkeey@gmail.com>
Signed-off-by: Christopher Blake <chrisrblake93@gmail.com>

arch/arm: Add __memset alias to memset_rpi.S

memset_rpi.S is an optimised memset implementation, but doesn't define
__memset (which was just added to memset.S). As a result, building
for the BCM2835 platform causes a link failure.

Add __memset as yet another alias to our common implementation.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>

arm: Fix custom rpi __memset32 and __memset64

See: #4798

Signed-off-by: Phil Elwell <phil@raspberrypi.com>

arm: Fix annoying .eh_frame section warnings

Replace the cfi directives with the UNWIND equivalents. This prevents
the .eh_frame section from being created, eliminating the warnings.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>

arch/arm/include/asm/string.h

@@ -65,4 +65,9 @@ static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n)
 
 #endif
 
+#ifdef CONFIG_BCM2835_FAST_MEMCPY
+#define __HAVE_ARCH_MEMCMP
+extern int memcmp(const void *, const void *, size_t);
+#endif
+
 #endif

arch/arm/include/asm/uaccess.h

@@ -500,6 +500,9 @@ do {									\
 extern unsigned long __must_check
 arm_copy_from_user(void *to, const void __user *from, unsigned long n);
 
+extern unsigned long __must_check
+__copy_from_user_std(void *to, const void __user *from, unsigned long n);
+
 static inline unsigned long __must_check
 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
 {

arch/arm/lib/Makefile

@@ -7,8 +7,8 @@
 
 lib-y		:= changebit.o csumipv6.o csumpartial.o               \
 		   csumpartialcopy.o csumpartialcopyuser.o clearbit.o \
-		   delay.o delay-loop.o findbit.o memchr.o memcpy.o   \
-		   memmove.o memset.o setbit.o                        \
+		   delay.o delay-loop.o findbit.o memchr.o            \
+		   setbit.o                                           \
 		   strchr.o strrchr.o                                 \
 		   testchangebit.o testclearbit.o testsetbit.o        \
 		   ashldi3.o ashrdi3.o lshrdi3.o muldi3.o             \
@@ -25,6 +25,16 @@ else
   lib-y	+= backtrace.o
 endif
 
+# Choose optimised implementations for Raspberry Pi
+ifeq ($(CONFIG_BCM2835_FAST_MEMCPY),y)
+  CFLAGS_uaccess_with_memcpy.o += -DCOPY_FROM_USER_THRESHOLD=1600
+  CFLAGS_uaccess_with_memcpy.o += -DCOPY_TO_USER_THRESHOLD=672
+  obj-$(CONFIG_MODULES) += exports_rpi.o
+  lib-y        += memcpy_rpi.o memmove_rpi.o memset_rpi.o memcmp_rpi.o
+else
+  lib-y        += memcpy.o memmove.o memset.o
+endif
+
 # using lib_ here won't override already available weak symbols
 obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
 

arch/arm/lib/arm-mem.h

@@ -0,0 +1,159 @@
+/*
+Copyright (c) 2013, Raspberry Pi Foundation
+Copyright (c) 2013, RISC OS Open Ltd
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the copyright holder nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+.macro myfunc fname
+ .func fname
+ .global fname
+fname:
+.endm
+
+.macro preload_leading_step1  backwards, ptr, base
+/* If the destination is already 16-byte aligned, then we need to preload
+ * between 0 and prefetch_distance (inclusive) cache lines ahead so there
+ * are no gaps when the inner loop starts.
+ */
+ .if backwards
+        sub     ptr, base, #1
+        bic     ptr, ptr, #31
+ .else
+        bic     ptr, base, #31
+ .endif
+ .set OFFSET, 0
+ .rept prefetch_distance+1
+        pld     [ptr, #OFFSET]
+  .if backwards
+   .set OFFSET, OFFSET-32
+  .else
+   .set OFFSET, OFFSET+32
+  .endif
+ .endr
+.endm
+
+.macro preload_leading_step2  backwards, ptr, base, leading_bytes, tmp
+/* However, if the destination is not 16-byte aligned, we may need to
+ * preload one more cache line than that. The question we need to ask is:
+ * are the leading bytes more than the amount by which the source
+ * pointer will be rounded down for preloading, and if so, by how many
+ * cache lines?
+ */
+ .if backwards
+/* Here we compare against how many bytes we are into the
+ * cache line, counting down from the highest such address.
+ * Effectively, we want to calculate
+ *     leading_bytes = dst&15
+ *     cacheline_offset = 31-((src-leading_bytes-1)&31)
+ *     extra_needed = leading_bytes - cacheline_offset
+ * and test if extra_needed is <= 0, or rearranging:
+ *     leading_bytes + (src-leading_bytes-1)&31 <= 31
+ */
+        mov     tmp, base, lsl #32-5
+        sbc     tmp, tmp, leading_bytes, lsl #32-5
+        adds    tmp, tmp, leading_bytes, lsl #32-5
+        bcc     61f
+        pld     [ptr, #-32*(prefetch_distance+1)]
+ .else
+/* Effectively, we want to calculate
+ *     leading_bytes = (-dst)&15
+ *     cacheline_offset = (src+leading_bytes)&31
+ *     extra_needed = leading_bytes - cacheline_offset
+ * and test if extra_needed is <= 0.
+ */
+        mov     tmp, base, lsl #32-5
+        add     tmp, tmp, leading_bytes, lsl #32-5
+        rsbs    tmp, tmp, leading_bytes, lsl #32-5
+        bls     61f
+        pld     [ptr, #32*(prefetch_distance+1)]
+ .endif
+61:
+.endm
+
+.macro preload_trailing  backwards, base, remain, tmp
+        /* We need either 0, 1 or 2 extra preloads */
+ .if backwards
+        rsb     tmp, base, #0
+        mov     tmp, tmp, lsl #32-5
+ .else
+        mov     tmp, base, lsl #32-5
+ .endif
+        adds    tmp, tmp, remain, lsl #32-5
+        adceqs  tmp, tmp, #0
+        /* The instruction above has two effects: ensures Z is only
+         * set if C was clear (so Z indicates that both shifted quantities
+         * were 0), and clears C if Z was set (so C indicates that the sum
+         * of the shifted quantities was greater and not equal to 32) */
+        beq     82f
+ .if backwards
+        sub     tmp, base, #1
+        bic     tmp, tmp, #31
+ .else
+        bic     tmp, base, #31
+ .endif
+        bcc     81f
+ .if backwards
+        pld     [tmp, #-32*(prefetch_distance+1)]
+81:
+        pld     [tmp, #-32*prefetch_distance]
+ .else
+        pld     [tmp, #32*(prefetch_distance+2)]
+81:
+        pld     [tmp, #32*(prefetch_distance+1)]
+ .endif
+82:
+.endm
+
+.macro preload_all    backwards, narrow_case, shift, base, remain, tmp0, tmp1
+ .if backwards
+        sub     tmp0, base, #1
+        bic     tmp0, tmp0, #31
+        pld     [tmp0]
+        sub     tmp1, base, remain, lsl #shift
+ .else
+        bic     tmp0, base, #31
+        pld     [tmp0]
+        add     tmp1, base, remain, lsl #shift
+        sub     tmp1, tmp1, #1
+ .endif
+        bic     tmp1, tmp1, #31
+        cmp     tmp1, tmp0
+        beq     92f
+ .if narrow_case
+        /* In this case, all the data fits in either 1 or 2 cache lines */
+        pld     [tmp1]
+ .else
+91:
+  .if backwards
+        sub     tmp0, tmp0, #32
+  .else
+        add     tmp0, tmp0, #32
+  .endif
+        cmp     tmp0, tmp1
+        pld     [tmp0]
+        bne     91b
+ .endif
+92:
+.endm

arch/arm/lib/copy_from_user.S

@@ -104,7 +104,8 @@ UNWIND( .save	{r0, r2, r3, \regs}		)
 
 	.text
 
-ENTRY(arm_copy_from_user)
+ENTRY(__copy_from_user_std)
+WEAK(arm_copy_from_user)
 #ifdef CONFIG_CPU_SPECTRE
 	ldr	r3, =TASK_SIZE
 	uaccess_mask_range_ptr r1, r2, r3, ip
@@ -113,6 +114,7 @@ ENTRY(arm_copy_from_user)
 #include "copy_template.S"
 
 ENDPROC(arm_copy_from_user)
+ENDPROC(__copy_from_user_std)
 
 	.pushsection .text.fixup,"ax"
 	.align 0

arch/arm/lib/exports_rpi.c

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2014, Raspberry Pi (Trading) Ltd.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2, as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+EXPORT_SYMBOL(memcmp);