-
Notifications
You must be signed in to change notification settings - Fork 4.6k
/
WriteBarriers.S
390 lines (324 loc) · 18.7 KB
/
WriteBarriers.S
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
.syntax unified
.thumb
#include <AsmOffsets.inc> // generated by the build from AsmOffsets.cpp
#include <unixasmmacros.inc>
#ifdef WRITE_BARRIER_CHECK
.macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG
// If g_GCShadow is 0, don't perform the check.
ldr r12, =C_FUNC(g_GCShadow)
ldr r12, [r12]
cbz r12, LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG)
// Save DESTREG since we're about to modify it (and we need the original value both within the macro and
// once we exit the macro). Note that this is naughty since we're altering the stack pointer outside of
// the prolog inside a method without a frame. But given that this is only debug code and generally we
// shouldn't be walking the stack at this point it seems preferable to recoding the all the barrier
// variants to set up frames. The compiler knows exactly which registers are trashed in the simple write
// barrier case, so we don't have any more scratch registers to play with (and doing so would only make
// things harder if at a later stage we want to allow multiple barrier versions based on the input
// registers).
push \DESTREG
// Transform DESTREG into the equivalent address in the shadow heap.
ldr r12, =C_FUNC(g_lowest_address)
ldr r12, [r12]
sub \DESTREG, r12
cmp \DESTREG, #0
blo LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
ldr r12, =C_FUNC(g_GCShadow)
ldr r12, [r12]
add \DESTREG, r12
ldr r12, =C_FUNC(g_GCShadowEnd)
ldr r12, [r12]
cmp \DESTREG, r12
bhs LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG)
// Update the shadow heap.
str \REFREG, [\DESTREG]
// The following read must be strongly ordered wrt to the write we've just performed in order to
// prevent race conditions.
dmb
// Now check that the real heap location still contains the value we just wrote into the shadow heap.
mov r12, \DESTREG
ldr \DESTREG, [sp]
str r12, [sp]
ldr r12, [\DESTREG]
cmp r12, \REFREG
bne LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG)
// The original DESTREG value is now restored but the stack has a value (the shadow version of the
// location) pushed. Need to discard this push before we are done.
add sp, #4
b LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG)
LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Invalidate_\REFREG):
// Someone went and updated the real heap. We need to invalidate the shadow location since we can't
// guarantee whose shadow update won.
// Retrieve shadow location from the stack and restore original DESTREG to the stack. This is an
// additional memory barrier we don't require but it's on the rare path and x86 doesn't have an xchg
// variant that doesn't implicitly specify the lock prefix. Note that INVALIDGCVALUE is a 32-bit
// immediate and therefore must be moved into a register before it can be written to the shadow
// location.
mov r12, \DESTREG
ldr \DESTREG, [sp]
str r12, [sp]
push \REFREG
movw \REFREG, #(INVALIDGCVALUE & 0xFFFF)
movt \REFREG, #(INVALIDGCVALUE >> 16)
str \REFREG, [\DESTREG]
pop \REFREG
LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_PopThenDone_\REFREG):
// Restore original DESTREG value from the stack.
pop \DESTREG
LOCAL_LABEL(\BASENAME\()_UpdateShadowHeap_Done_\REFREG):
.endm
#else // WRITE_BARRIER_CHECK
.macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG
.endm
#endif // WRITE_BARRIER_CHECK
// There are several different helpers used depending on which register holds the object reference. Since all
// the helpers have identical structure we use a macro to define this structure. Two arguments are taken, the
// name of the register that points to the location to be updated and the name of the register that holds the
// object reference (this should be in upper case as it's used in the definition of the name of the helper).
.macro DEFINE_UNCHECKED_WRITE_BARRIER_CORE BASENAME, REFREG
// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW \BASENAME, \REFREG, r0
// If the reference is to an object that's not in an ephemeral generation we have no need to track it
// (since the object won't be collected or moved by an ephemeral collection).
ldr r12, =C_FUNC(g_ephemeral_low)
ldr r12, [r12]
cmp \REFREG, r12
blo LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)
ldr r12, =C_FUNC(g_ephemeral_high)
ldr r12, [r12]
cmp \REFREG, r12
bhs LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)
// We have a location on the GC heap being updated with a reference to an ephemeral object so we must
// track this write. The location address is translated into an offset in the card table bitmap. We set
// an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
// the byte if it hasn't already been done since writes are expensive and impact scaling.
ldr r12, =C_FUNC(g_card_table)
ldr r12, [r12]
add r0, r12, r0, lsr #LOG2_CLUMP_SIZE
ldrb r12, [r0]
cmp r12, #0x0FF
bne LOCAL_LABEL(\BASENAME\()_UpdateCardTable_\REFREG)
LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG):
b LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG)
// We get here if it's necessary to update the card table.
LOCAL_LABEL(\BASENAME\()_UpdateCardTable_\REFREG):
mov r12, #0x0FF
strb r12, [r0]
LOCAL_LABEL(\BASENAME\()_EXIT_\REFREG):
.endm
// There are several different helpers used depending on which register holds the object reference. Since all
// the helpers have identical structure we use a macro to define this structure. One argument is taken, the
// name of the register that will hold the object reference (this should be in upper case as it's used in the
// definition of the name of the helper).
.macro DEFINE_UNCHECKED_WRITE_BARRIER REFREG, EXPORT_REG_NAME
// Define a helper with a name of the form RhpAssignRefEAX etc. (along with suitable calling standard
// decoration). The location to be updated is in DESTREG. The object reference that will be assigned into that
// location is in one of the other general registers determined by the value of REFREG.
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at WriteBarrierFunctionAvLOC
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
LEAF_ENTRY RhpAssignRef\EXPORT_REG_NAME, _TEXT
// Export the canonical write barrier under unqualified name as well
.ifc \REFREG, r1
ALTERNATE_ENTRY RhpAssignRef
.endif
// Use the GC write barrier as a convenient place to implement the managed memory model for ARM. The
// intent is that writes to the target object ($REFREG) will be visible across all CPUs before the
// write to the destination ($DESTREG). This covers most of the common scenarios where the programmer
// might assume strongly ordered accessess, namely where the preceding writes are used to initialize
// the object and the final write, made by this barrier in the instruction following the DMB,
// publishes that object for other threads/cpus to see.
//
// Note that none of this is relevant for single cpu machines. We may choose to implement a
// uniprocessor specific version of this barrier if uni-proc becomes a significant scenario again.
dmb
// Write the reference into the location. Note that we rely on the fact that no GC can occur between here
// and the card table update we may perform below.
ALTERNATE_ENTRY "RhpAssignRefAvLocation"\EXPORT_REG_NAME // WriteBarrierFunctionAvLocation
.ifc \REFREG, r1
ALTERNATE_ENTRY RhpAssignRefAVLocation
.endif
str \REFREG, [r0]
DEFINE_UNCHECKED_WRITE_BARRIER_CORE RhpAssignRef, \REFREG
bx lr
LEAF_END RhpAssignRef\EXPORT_REG_NAME, _TEXT
.endm
// One day we might have write barriers for all the possible argument registers but for now we have
// just one write barrier that assumes the input register is RSI.
DEFINE_UNCHECKED_WRITE_BARRIER r1, r1
//
// Define the helpers used to implement the write barrier required when writing an object reference into a
// location residing on the GC heap. Such write barriers allow the GC to optimize which objects in
// non-ephemeral generations need to be scanned for references to ephemeral objects during an ephemeral
// collection.
//
.macro DEFINE_CHECKED_WRITE_BARRIER_CORE BASENAME, REFREG
// The location being updated might not even lie in the GC heap (a handle or stack location for instance),
// in which case no write barrier is required.
ldr r12, =C_FUNC(g_lowest_address)
ldr r12, [r12]
cmp r0, r12
blo LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG)
ldr r12, =C_FUNC(g_highest_address)
ldr r12, [r12]
cmp r0, r12
bhs LOCAL_LABEL(\BASENAME\()_NoBarrierRequired_\REFREG)
DEFINE_UNCHECKED_WRITE_BARRIER_CORE \BASENAME, \REFREG
.endm
// There are several different helpers used depending on which register holds the object reference. Since all
// the helpers have identical structure we use a macro to define this structure. One argument is taken, the
// name of the register that will hold the object reference (this should be in upper case as it's used in the
// definition of the name of the helper).
.macro DEFINE_CHECKED_WRITE_BARRIER REFREG, EXPORT_REG_NAME
// Define a helper with a name of the form RhpCheckedAssignRefEAX etc. (along with suitable calling standard
// decoration). The location to be updated is always in R0. The object reference that will be assigned into
// that location is in one of the other general registers determined by the value of REFREG.
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen on the first instruction
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
LEAF_ENTRY RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT
// Export the canonical write barrier under unqualified name as well
.ifc \REFREG, r1
ALTERNATE_ENTRY RhpCheckedAssignRef
.endif
// Use the GC write barrier as a convenient place to implement the managed memory model for ARM. The
// intent is that writes to the target object ($REFREG) will be visible across all CPUs before the
// write to the destination ($DESTREG). This covers most of the common scenarios where the programmer
// might assume strongly ordered accessess, namely where the preceding writes are used to initialize
// the object and the final write, made by this barrier in the instruction following the DMB,
// publishes that object for other threads/cpus to see.
//
// Note that none of this is relevant for single cpu machines. We may choose to implement a
// uniprocessor specific version of this barrier if uni-proc becomes a significant scenario again.
dmb
// Write the reference into the location. Note that we rely on the fact that no GC can occur between here
// and the card table update we may perform below.
ALTERNATE_ENTRY "RhpCheckedAssignRefAvLocation"\EXPORT_REG_NAME // WriteBarrierFunctionAvLocation
.ifc \REFREG, r1
ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
.endif
str \REFREG, [r0]
DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedAssignRef, \REFREG
bx lr
LEAF_END RhpCheckedAssignRef\EXPORT_REG_NAME, _TEXT
.endm
// One day we might have write barriers for all the possible argument registers but for now we have
// just one write barrier that assumes the input register is RSI.
DEFINE_CHECKED_WRITE_BARRIER r1, r1
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpCheckedLockCmpXchgAVLocation
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
// r0 = destination address
// r1 = value
// r2 = comparand
LEAF_ENTRY RhpCheckedLockCmpXchg, _TEXT
// To implement our chosen memory model for ARM we insert a memory barrier at GC write brriers. This
// barrier must occur before the object reference update, so we have to do it unconditionally even
// though the update may fail below.
dmb
ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
LOCAL_LABEL(RhpCheckedLockCmpXchgRetry):
ldrex r3, [r0]
cmp r2, r3
bne LOCAL_LABEL(RhpCheckedLockCmpXchg_NoBarrierRequired_r1)
strex r3, r1, [r0]
cmp r3, #0
bne LOCAL_LABEL(RhpCheckedLockCmpXchgRetry)
mov r3, r2
DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, r1
mov r0, r3
bx lr
LEAF_END RhpCheckedLockCmpXchg, _TEXT
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpCheckedXchgAVLocation
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
// r0 = destination address
// r1 = value
LEAF_ENTRY RhpCheckedXchg, _TEXT
// To implement our chosen memory model for ARM we insert a memory barrier at GC write barriers. This
// barrier must occur before the object reference update.
dmb
ALTERNATE_ENTRY RhpCheckedXchgAVLocation
LOCAL_LABEL(RhpCheckedXchgRetry):
ldrex r2, [r0]
strex r3, r1, [r0]
cmp r3, #0
bne LOCAL_LABEL(RhpCheckedXchgRetry)
DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, r1
// The original value is currently in r2. We need to return it in r0.
mov r0, r2
bx lr
LEAF_END RhpCheckedXchg, _TEXT
//
// RhpByRefAssignRef simulates movs instruction for object references.
//
// On entry:
// r0: address of ref-field (assigned to)
// r1: address of the data (source)
// r2, r3: be trashed
//
// On exit:
// r0, r1 are incremented by 4,
// r2, r3: trashed
//
// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2
// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
LEAF_ENTRY RhpByRefAssignRef, _TEXT
// See comment in RhpAssignRef
dmb
ALTERNATE_ENTRY RhpByRefAssignRefAVLocation1
ldr r2, [r1]
ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2
str r2, [r0]
// Check whether the writes were even into the heap. If not there's no card update required.
ldr r3, =C_FUNC(g_lowest_address)
ldr r3, [r3]
cmp r0, r3
blo LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
ldr r3, =C_FUNC(g_highest_address)
ldr r3, [r3]
cmp r0, r3
bhs LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW BASENAME, r2, r0
// If the reference is to an object that's not in an ephemeral generation we have no need to track it
// (since the object won't be collected or moved by an ephemeral collection).
ldr r3, =C_FUNC(g_ephemeral_low)
ldr r3, [r3]
cmp r2, r3
blo LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
ldr r3, =C_FUNC(g_ephemeral_high)
ldr r3, [r3]
cmp r2, r3
bhs LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
// move current r0 value into r2 and then increment the pointers
mov r2, r0
add r1, #4
add r0, #4
// We have a location on the GC heap being updated with a reference to an ephemeral object so we must
// track this write. The location address is translated into an offset in the card table bitmap. We set
// an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
// the byte if it hasn't already been done since writes are expensive and impact scaling.
ldr r3, =C_FUNC(g_card_table)
ldr r3, [r3]
add r2, r3, r2, lsr #LOG2_CLUMP_SIZE
ldrb r3, [r2]
cmp r3, #0x0FF
bne LOCAL_LABEL(RhpByRefAssignRef_UpdateCardTable)
bx lr
// We get here if it's necessary to update the card table.
LOCAL_LABEL(RhpByRefAssignRef_UpdateCardTable):
mov r3, #0x0FF
strb r3, [r2]
bx lr
LOCAL_LABEL(RhpByRefAssignRef_NotInHeap):
// Increment the pointers before leaving
add r0, #4
add r1, #4
bx lr
LEAF_END RhpByRefAssignRef, _TEXT