-
Notifications
You must be signed in to change notification settings - Fork 10.8k
/
memmove.cpp
119 lines (110 loc) · 4.59 KB
/
memmove.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
//===-- Implementation of memmove -----------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "src/string/memmove.h"
#include "src/__support/common.h"
#include "src/string/memory_utils/op_aarch64.h"
#include "src/string/memory_utils/op_builtin.h"
#include "src/string/memory_utils/op_generic.h"
#include "src/string/memory_utils/op_x86.h"
#include <stddef.h> // size_t, ptrdiff_t
#include <stdio.h>
namespace __llvm_libc {
[[maybe_unused]] static inline void
inline_memmove_embedded_tiny(Ptr dst, CPtr src, size_t count) {
if ((count == 0) || (dst == src))
return;
if (dst < src) {
#pragma nounroll
for (size_t offset = 0; offset < count; ++offset)
builtin::Memcpy<1>::block(dst + offset, src + offset);
} else {
#pragma nounroll
for (ptrdiff_t offset = count - 1; offset >= 0; --offset)
builtin::Memcpy<1>::block(dst + offset, src + offset);
}
}
template <size_t MaxSize>
[[maybe_unused]] static inline void inline_memmove_generic(Ptr dst, CPtr src,
size_t count) {
if (count == 0)
return;
if (count == 1)
return generic::Memmove<1, MaxSize>::block(dst, src);
if (count <= 4)
return generic::Memmove<2, MaxSize>::head_tail(dst, src, count);
if (count <= 8)
return generic::Memmove<4, MaxSize>::head_tail(dst, src, count);
if (count <= 16)
return generic::Memmove<8, MaxSize>::head_tail(dst, src, count);
if (count <= 32)
return generic::Memmove<16, MaxSize>::head_tail(dst, src, count);
if (count <= 64)
return generic::Memmove<32, MaxSize>::head_tail(dst, src, count);
if (count <= 128)
return generic::Memmove<64, MaxSize>::head_tail(dst, src, count);
if (dst < src) {
generic::Memmove<32, MaxSize>::template align_forward<Arg::Src>(dst, src,
count);
return generic::Memmove<64, MaxSize>::loop_and_tail_forward(dst, src,
count);
} else {
generic::Memmove<32, MaxSize>::template align_backward<Arg::Src>(dst, src,
count);
return generic::Memmove<64, MaxSize>::loop_and_tail_backward(dst, src,
count);
}
}
static inline void inline_memmove(Ptr dst, CPtr src, size_t count) {
#if defined(LLVM_LIBC_ARCH_X86) || defined(LLVM_LIBC_ARCH_AARCH64)
#if defined(LLVM_LIBC_ARCH_X86)
static constexpr size_t kMaxSize = x86::kAvx512F ? 64
: x86::kAvx ? 32
: x86::kSse2 ? 16
: 8;
#elif defined(LLVM_LIBC_ARCH_AARCH64)
static constexpr size_t kMaxSize = aarch64::kNeon ? 16 : 8;
#endif
// return inline_memmove_generic<kMaxSize>(dst, src, count);
if (count == 0)
return;
if (count == 1)
return generic::Memmove<1, kMaxSize>::block(dst, src);
if (count <= 4)
return generic::Memmove<2, kMaxSize>::head_tail(dst, src, count);
if (count <= 8)
return generic::Memmove<4, kMaxSize>::head_tail(dst, src, count);
if (count <= 16)
return generic::Memmove<8, kMaxSize>::head_tail(dst, src, count);
if (count <= 32)
return generic::Memmove<16, kMaxSize>::head_tail(dst, src, count);
if (count <= 64)
return generic::Memmove<32, kMaxSize>::head_tail(dst, src, count);
if (count <= 128)
return generic::Memmove<64, kMaxSize>::head_tail(dst, src, count);
if (dst < src) {
generic::Memmove<32, kMaxSize>::align_forward<Arg::Src>(dst, src, count);
return generic::Memmove<64, kMaxSize>::loop_and_tail_forward(dst, src,
count);
} else {
generic::Memmove<32, kMaxSize>::align_backward<Arg::Src>(dst, src, count);
return generic::Memmove<64, kMaxSize>::loop_and_tail_backward(dst, src,
count);
}
#elif defined(LLVM_LIBC_ARCH_ARM)
return inline_memmove_embedded_tiny(dst, src, count);
#else
#error "Unsupported platform"
#endif
}
LLVM_LIBC_FUNCTION(void *, memmove,
(void *dst, const void *src, size_t count)) {
inline_memmove(reinterpret_cast<Ptr>(dst), reinterpret_cast<CPtr>(src),
count);
return dst;
}
} // namespace __llvm_libc