/
Malloc.h
221 lines (193 loc) · 6.52 KB
/
Malloc.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
/*
* Copyright 2012 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Functions to provide smarter use of jemalloc, if jemalloc is being used.
// http://www.canonware.com/download/jemalloc/jemalloc-latest/doc/jemalloc.html
#ifndef FOLLY_MALLOC_H_
#define FOLLY_MALLOC_H_
// If using fbstring from libstdc++, then just define stub code
// here to typedef the fbstring type into the folly namespace.
// This provides backwards compatibility for code that explicitly
// includes and uses fbstring.
#if defined(_GLIBCXX_USE_FB) && !defined(_LIBSTDCXX_FBSTRING)
#include <string>
namespace folly {
using std::goodMallocSize;
using std::jemallocMinInPlaceExpandable;
using std::usingJEMalloc;
using std::smartRealloc;
using std::checkedMalloc;
using std::checkedCalloc;
using std::checkedRealloc;
}
#else // !defined(_GLIBCXX_USE_FB) || defined(_LIBSTDCXX_FBSTRING)
#ifdef _LIBSTDCXX_FBSTRING
#pragma GCC system_header
#define FOLLY_HAVE_MALLOC_H 1
#else
#include "folly-config.h"
#endif
// for malloc_usable_size
// NOTE: FreeBSD 9 doesn't have malloc.h. It's defitions
// are found in stdlib.h.
#ifdef FOLLY_HAVE_MALLOC_H
#include <malloc.h>
#else
#include <stdlib.h>
#endif
#include <cassert>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <new>
#include <bits/functexcept.h>
/**
* Declare rallocm() and malloc_usable_size() as weak symbols. It
* will be provided by jemalloc if we are using jemalloc, or it will
* be NULL if we are using another malloc implementation.
*/
extern "C" int rallocm(void**, size_t*, size_t, size_t, int)
__attribute__((weak));
/**
* Define the ALLOCM_SUCCESS, ALLOCM_ZERO, and ALLOCM_NO_MOVE constants
* normally provided by jemalloc. We define them so that we don't have to
* include jemalloc.h, in case the program is built without jemalloc support.
*/
#ifndef ALLOCM_SUCCESS
#define ALLOCM_SUCCESS 0
#define ALLOCM_ERR_OOM 1
#define ALLOCM_ERR_NOT_MOVED 2
#define ALLOCM_ZERO 64
#define ALLOCM_NO_MOVE 128
#endif
#ifdef _LIBSTDCXX_FBSTRING
namespace std _GLIBCXX_VISIBILITY(default) {
_GLIBCXX_BEGIN_NAMESPACE_VERSION
#else
namespace folly {
#endif
/**
* Determine if we are using jemalloc or not.
*/
inline bool usingJEMalloc() {
return rallocm != NULL;
}
/**
* For jemalloc's size classes, see
* http://www.canonware.com/download/jemalloc/jemalloc-latest/doc/jemalloc.html
*/
inline size_t goodMallocSize(size_t minSize) {
if (!usingJEMalloc()) {
// Not using jemalloc - no smarts
return minSize;
}
if (minSize <= 64) {
// Choose smallest allocation to be 64 bytes - no tripping over
// cache line boundaries, and small string optimization takes care
// of short strings anyway.
return 64;
}
if (minSize <= 512) {
// Round up to the next multiple of 64; we don't want to trip over
// cache line boundaries.
return (minSize + 63) & ~size_t(63);
}
if (minSize <= 3840) {
// Round up to the next multiple of 256
return (minSize + 255) & ~size_t(255);
}
if (minSize <= 4072 * 1024) {
// Round up to the next multiple of 4KB
return (minSize + 4095) & ~size_t(4095);
}
// Holy Moly
// Round up to the next multiple of 4MB
return (minSize + 4194303) & ~size_t(4194303);
}
// We always request "good" sizes for allocation, so jemalloc can
// never grow in place small blocks; they're already occupied to the
// brim. Blocks larger than or equal to 4096 bytes can in fact be
// expanded in place, and this constant reflects that.
static const size_t jemallocMinInPlaceExpandable = 4096;
/**
* Trivial wrappers around malloc, calloc, realloc that check for allocation
* failure and throw std::bad_alloc in that case.
*/
inline void* checkedMalloc(size_t size) {
void* p = malloc(size);
if (!p) std::__throw_bad_alloc();
return p;
}
inline void* checkedCalloc(size_t n, size_t size) {
void* p = calloc(n, size);
if (!p) std::__throw_bad_alloc();
return p;
}
inline void* checkedRealloc(void* ptr, size_t size) {
void* p = realloc(ptr, size);
if (!p) std::__throw_bad_alloc();
return p;
}
/**
* This function tries to reallocate a buffer of which only the first
* currentSize bytes are used. The problem with using realloc is that
* if currentSize is relatively small _and_ if realloc decides it
* needs to move the memory chunk to a new buffer, then realloc ends
* up copying data that is not used. It's impossible to hook into
* GNU's malloc to figure whether expansion will occur in-place or as
* a malloc-copy-free troika. (If an expand_in_place primitive would
* be available, smartRealloc would use it.) As things stand, this
* routine just tries to call realloc() (thus benefitting of potential
* copy-free coalescing) unless there's too much slack memory.
*/
inline void* smartRealloc(void* p,
const size_t currentSize,
const size_t currentCapacity,
const size_t newCapacity) {
assert(p);
assert(currentSize <= currentCapacity &&
currentCapacity < newCapacity);
if (usingJEMalloc()) {
// using jemalloc's API. Don't forget that jemalloc can never grow
// in place blocks smaller than 4096 bytes.
if (currentCapacity >= jemallocMinInPlaceExpandable &&
rallocm(&p, NULL, newCapacity, 0, ALLOCM_NO_MOVE) == ALLOCM_SUCCESS) {
// Managed to expand in place
return p;
}
// Cannot expand; must move
auto const result = checkedMalloc(newCapacity);
std::memcpy(result, p, currentSize);
free(p);
return result;
}
// No jemalloc no honey
auto const slack = currentCapacity - currentSize;
if (slack * 2 > currentSize) {
// Too much slack, malloc-copy-free cycle:
auto const result = checkedMalloc(newCapacity);
std::memcpy(result, p, currentSize);
free(p);
return result;
}
// If there's not too much slack, we realloc in hope of coalescing
return checkedRealloc(p, newCapacity);
}
#ifdef _LIBSTDCXX_FBSTRING
_GLIBCXX_END_NAMESPACE_VERSION
#endif
} // folly
#endif // !defined(_GLIBCXX_USE_FB) || defined(_LIBSTDCXX_FBSTRING)
#endif // FOLLY_MALLOC_H_