-
Notifications
You must be signed in to change notification settings - Fork 4.6k
/
WriteBarriers.S
371 lines (297 loc) · 12.3 KB
/
WriteBarriers.S
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#include <unixasmmacros.inc>
// Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used
// during garbage collections to verify that object references where never written to the heap without using a
// write barrier. Note that we are potentially racing to update the shadow heap while other threads are writing
// new references to the real heap. Since this can not be solved perfectly without critical sections around the
// entire update process, we instead update the shadow location and then re-check the real location (as two
// ordered operations) and if there is a disparity we will re-write the shadow location with a special value
// (INVALIDGCVALUE) which disables the check for that location. Since the shadow heap is only validated at GC
// time and these write barrier operations are atomic wrt to GCs this is sufficient to guarantee that the
// shadow heap contains only valid copies of real heap values or INVALIDGCVALUE.
#ifdef WRITE_BARRIER_CHECK
.global $g_GCShadow
.global $g_GCShadowEnd
// On entry:
// $destReg: location to be updated
// $refReg: objectref to be stored
//
// On exit:
// x9,x10: trashed
// other registers are preserved
//
.macro UPDATE_GC_SHADOW destReg, refReg
// If g_GCShadow is 0, don't perform the check.
PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9
cbz x9, 1f
// Save destReg since we're about to modify it (and we need the original value both within the macro and
// once we exit the macro).
mov x10, \destReg
// Transform destReg into the equivalent address in the shadow heap.
PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, X9
subs \destReg, \destReg, x9
blo 0f
PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9
add \destReg, \destReg, x9
PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, X9
cmp \destReg, x9
bhs 0f
// Update the shadow heap.
str \refReg, [\destReg]
// The following read must be strongly ordered wrt to the write we have just performed in order to
// prevent race conditions.
dmb ish
// Now check that the real heap location still contains the value we just wrote into the shadow heap.
mov x9, x10
ldr x9, [x9]
cmp x9, \refReg
beq 0f
// Someone went and updated the real heap. We need to invalidate INVALIDGCVALUE the shadow location since we can not
// guarantee whose shadow update won.
movz x9, (INVALIDGCVALUE & 0xFFFF) // #0xcccd
movk x9, ((INVALIDGCVALUE >> 16) & 0xFFFF), LSL #16
str x9, [\destReg]
0:
// Restore original destReg value
mov \destReg, x10
1:
.endm
#else // WRITE_BARRIER_CHECK
.macro UPDATE_GC_SHADOW destReg, refReg
.endm
#endif // WRITE_BARRIER_CHECK
// There are several different helpers used depending on which register holds the object reference. Since all
// the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the
// name of the register that points to the location to be updated and the name of the register that holds the
// object reference (this should be in upper case as it is used in the definition of the name of the helper).
// Define a sub-macro first that expands to the majority of the barrier implementation. This is used below for
// some interlocked helpers that need an inline barrier.
// On entry:
// destReg: location to be updated
// refReg: objectref to be stored
// trash: register nr than can be trashed
//
// On exit:
// destReg: trashed
//
.macro INSERT_UNCHECKED_WRITE_BARRIER_CORE destReg, refReg, trash
// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we are in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW \destReg, \refReg
#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
// Update the write watch table if necessary
PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x\trash
cbz x\trash, 2f
add x\trash, x\trash, \destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift
ldrb w17, [x\trash]
cbnz x17, 2f
mov w17, #0xFF
strb w17, [x\trash]
#endif
2:
// We can skip the card table write if the reference is to
// an object not on the epehemeral segment.
PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x\trash
cmp \refReg, x\trash
blo 0f
PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x\trash
cmp \refReg, x\trash
bhs 0f
// Set this objects card, if it has not already been set.
PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x\trash
add x17, x\trash, \destReg, lsr #11
// Check that this card has not already been written. Avoiding useless writes is a big win on
// multi-proc systems since it avoids cache thrashing.
ldrb w\trash, [x17]
cmp x\trash, 0xFF
beq 0f
mov x\trash, 0xFF
strb w\trash, [x17]
#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
// Check if we need to update the card bundle table
PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x\trash
add x17, x\trash, \destReg, lsr #21
ldrb w\trash, [x17]
cmp x\trash, 0xFF
beq 0f
mov x\trash, 0xFF
strb w\trash, [x17]
#endif
0:
// Exit label
.endm
// On entry:
// destReg: location to be updated
// refReg: objectref to be stored
// trash: register nr than can be trashed
//
// On exit:
// destReg: trashed
//
.macro INSERT_CHECKED_WRITE_BARRIER_CORE destReg, refReg, trash
// The "check" of this checked write barrier - is destReg
// within the heap? if no, early out.
PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x\trash
cmp \destReg, x\trash
PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x\trash
// If \destReg >= g_lowest_address, compare \destReg to g_highest_address.
// Otherwise, set the C flag (0x2) to take the next branch.
ccmp \destReg, x\trash, #0x2, hs
bhs 0f
INSERT_UNCHECKED_WRITE_BARRIER_CORE \destReg, \refReg, \trash
0:
// Exit label
.endm
// RhpCheckedAssignRef(Object** dst, Object* src)
//
// Write barrier for writes to objects that may reside
// on the managed heap.
//
// On entry:
// x0 : the destination address (LHS of the assignment).
// May not be an object reference (hence the checked).
// x1 : the object reference (RHS of the assignment).
// On exit:
// x1 : trashed
// x9 : trashed
LEAF_ENTRY RhpCheckedAssignRef, _TEXT
ALTERNATE_ENTRY RhpCheckedAssignRefX1
mov x14, x0 ; x14 = dst
mov x15, x1 ; x15 = val
b C_FUNC(RhpCheckedAssignRefArm64)
LEAF_END RhpCheckedAssignRef, _TEXT
// RhpAssignRef(Object** dst, Object* src)
//
// Write barrier for writes to objects that are known to
// reside on the managed heap.
//
// On entry:
// x0 : the destination address (LHS of the assignment).
// x1 : the object reference (RHS of the assignment).
// On exit:
// x1 : trashed
// x9 : trashed
LEAF_ENTRY RhpAssignRef, _TEXT
ALTERNATE_ENTRY RhpAssignRefX1
mov x14, x0 ; x14 = dst
mov x15, x1 ; x15 = val
b C_FUNC(RhpAssignRefArm64)
LEAF_END RhpAssignRef, _TEXT
// Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon
// successful updates.
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpCheckedLockCmpXchgAVLocation
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
// RhpCheckedLockCmpXchg(Object** dest, Object* value, Object* comparand)
//
// Interlocked compare exchange on objectref.
//
// On entry:
// x0: pointer to objectref
// x1: exchange value
// x2: comparand
//
// On exit:
// x0: original value of objectref
// x9: trashed
// x10: trashed
//
LEAF_ENTRY RhpCheckedLockCmpXchg
ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
CmpXchgRetry:
// Check location value is what we expect.
ldaxr x10, [x0]
cmp x10, x2
bne CmpXchgNoUpdate
// Current value matches comparand, attempt to update with the new value.
stlxr w9, x1, [x0]
cbnz w9, CmpXchgRetry
// We have successfully updated the value of the objectref so now we need a GC write barrier.
// The following barrier code takes the destination in x0 and the value in x1 so the arguments are
// already correctly set up.
INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9
CmpXchgNoUpdate:
// x10 still contains the original value.
mov x0, x10
ArmInterlockedOperationBarrier
ret lr
LEAF_END RhpCheckedLockCmpXchg, _TEXT
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen within at RhpCheckedXchgAVLocation
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
// RhpCheckedXchg(Object** destination, Object* value)
//
// Interlocked exchange on objectref.
//
// On entry:
// x0: pointer to objectref
// x1: exchange value
//
// On exit:
// x0: original value of objectref
// x9: trashed
// x10: trashed
//
LEAF_ENTRY RhpCheckedXchg, _TEXT
ALTERNATE_ENTRY RhpCheckedXchgAVLocation
ExchangeRetry:
// Read the existing memory location.
ldaxr x10, [x0]
// Attempt to update with the new value.
stlxr w9, x1, [x0]
cbnz w9, ExchangeRetry
// We have successfully updated the value of the objectref so now we need a GC write barrier.
// The following barrier code takes the destination in x0 and the value in x1 so the arguments are
// already correctly set up.
INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9
// x10 still contains the original value.
mov x0, x10
ArmInterlockedOperationBarrier
ret
LEAF_END RhpCheckedXchg, _TEXT
LEAF_ENTRY RhpAssignRefArm64, _TEXT
ALTERNATE_ENTRY RhpAssignRefAVLocation
ALTERNATE_ENTRY RhpAssignRefX1AVLocation
stlr x15, [x14]
INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15, 12
ret
LEAF_END RhpAssignRefArm64, _TEXT
// void JIT_CheckedWriteBarrier(Object** dst, Object* src)
// On entry:
// x14 : the destination address (LHS of the assignment)
// x15 : the object reference (RHS of the assignment)
//
// On exit:
// x12 : trashed
// x14 : trashed (incremented by 8 to implement JIT_ByRefWriteBarrier contract)
// x15 : trashed
// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
//
LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation
stlr x15, [x14]
INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, 12
ret
LEAF_END RhpCheckedAssignRefArm64, _TEXT
// void JIT_ByRefWriteBarrier
// On entry:
// x13 : the source address (points to object reference to write)
// x14 : the destination address (object reference written here)
//
// On exit:
// x12 : trashed
// x13 : incremented by 8
// x14 : incremented by 8
// x15 : trashed
// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
//
LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
ldr x15, [x13]
stlr x15, [x14]
INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, 12
add X13, x13, #8
add x14, x14, #8
ret
LEAF_END RhpByRefAssignRefArm64, _TEXT