1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7#include "funcdata.h"
8
9// bool armcas(int32 *val, int32 old, int32 new)
10// Atomically:
11// if(*val == old){
12// *val = new;
13// return 1;
14// }else
15// return 0;
16//
17// To implement ·cas in sys_$GOOS_arm.s
18// using the native instructions, use:
19//
20// TEXT ·cas(SB),NOSPLIT,$0
21// B ·armcas(SB)
22//
23TEXT ·armcas(SB),NOSPLIT,$0-13
24 MOVW ptr+0(FP), R1
25 MOVW old+4(FP), R2
26 MOVW new+8(FP), R3
27casl:
28 LDREX (R1), R0
29 CMP R0, R2
30 BNE casfail
31
32#ifndef GOARM_7
33 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
34 CMP $0, R11
35 BEQ 2(PC)
36#endif
37 DMB MB_ISHST
38
39 STREX R3, (R1), R0
40 CMP $0, R0
41 BNE casl
42 MOVW $1, R0
43
44#ifndef GOARM_7
45 CMP $0, R11
46 BEQ 2(PC)
47#endif
48 DMB MB_ISH
49
50 MOVB R0, ret+12(FP)
51 RET
52casfail:
53 MOVW $0, R0
54 MOVB R0, ret+12(FP)
55 RET
56
57// stubs
58
59TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
60 B ·Load(SB)
61
62TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
63 B ·Load(SB)
64
65TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
66 B ·Load(SB)
67
68TEXT ·Casint32(SB),NOSPLIT,$0-13
69 B ·Cas(SB)
70
71TEXT ·Casint64(SB),NOSPLIT,$-4-21
72 B ·Cas64(SB)
73
74TEXT ·Casuintptr(SB),NOSPLIT,$0-13
75 B ·Cas(SB)
76
77TEXT ·Casp1(SB),NOSPLIT,$0-13
78 B ·Cas(SB)
79
80TEXT ·CasRel(SB),NOSPLIT,$0-13
81 B ·Cas(SB)
82
83TEXT ·Loadint32(SB),NOSPLIT,$0-8
84 B ·Load(SB)
85
86TEXT ·Loadint64(SB),NOSPLIT,$-4-12
87 B ·Load64(SB)
88
89TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
90 B ·Load(SB)
91
92TEXT ·Loaduint(SB),NOSPLIT,$0-8
93 B ·Load(SB)
94
95TEXT ·Storeint32(SB),NOSPLIT,$0-8
96 B ·Store(SB)
97
98TEXT ·Storeint64(SB),NOSPLIT,$0-12
99 B ·Store64(SB)
100
101TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
102 B ·Store(SB)
103
104TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
105 B ·Store(SB)
106
107TEXT ·StoreRel(SB),NOSPLIT,$0-8
108 B ·Store(SB)
109
110TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
111 B ·Store(SB)
112
113TEXT ·Xaddint32(SB),NOSPLIT,$0-12
114 B ·Xadd(SB)
115
116TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
117 B ·Xadd64(SB)
118
119TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
120 B ·Xadd(SB)
121
122TEXT ·Xchgint32(SB),NOSPLIT,$0-12
123 B ·Xchg(SB)
124
125TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
126 B ·Xchg64(SB)
127
128// 64-bit atomics
129// The native ARM implementations use LDREXD/STREXD, which are
130// available on ARMv6k or later. We use them only on ARMv7.
131// On older ARM, we use Go implementations which simulate 64-bit
132// atomics with locks.
133TEXT armCas64<>(SB),NOSPLIT,$0-21
134 // addr is already in R1
135 MOVW old_lo+4(FP), R2
136 MOVW old_hi+8(FP), R3
137 MOVW new_lo+12(FP), R4
138 MOVW new_hi+16(FP), R5
139cas64loop:
140 LDREXD (R1), R6 // loads R6 and R7
141 CMP R2, R6
142 BNE cas64fail
143 CMP R3, R7
144 BNE cas64fail
145
146 DMB MB_ISHST
147
148 STREXD R4, (R1), R0 // stores R4 and R5
149 CMP $0, R0
150 BNE cas64loop
151 MOVW $1, R0
152
153 DMB MB_ISH
154
155 MOVBU R0, swapped+20(FP)
156 RET
157cas64fail:
158 MOVW $0, R0
159 MOVBU R0, swapped+20(FP)
160 RET
161
162TEXT armXadd64<>(SB),NOSPLIT,$0-20
163 // addr is already in R1
164 MOVW delta_lo+4(FP), R2
165 MOVW delta_hi+8(FP), R3
166
167add64loop:
168 LDREXD (R1), R4 // loads R4 and R5
169 ADD.S R2, R4
170 ADC R3, R5
171
172 DMB MB_ISHST
173
174 STREXD R4, (R1), R0 // stores R4 and R5
175 CMP $0, R0
176 BNE add64loop
177
178 DMB MB_ISH
179
180 MOVW R4, new_lo+12(FP)
181 MOVW R5, new_hi+16(FP)
182 RET
183
184TEXT armXchg64<>(SB),NOSPLIT,$0-20
185 // addr is already in R1
186 MOVW new_lo+4(FP), R2
187 MOVW new_hi+8(FP), R3
188
189swap64loop:
190 LDREXD (R1), R4 // loads R4 and R5
191
192 DMB MB_ISHST
193
194 STREXD R2, (R1), R0 // stores R2 and R3
195 CMP $0, R0
196 BNE swap64loop
197
198 DMB MB_ISH
199
200 MOVW R4, old_lo+12(FP)
201 MOVW R5, old_hi+16(FP)
202 RET
203
204TEXT armLoad64<>(SB),NOSPLIT,$0-12
205 // addr is already in R1
206
207 LDREXD (R1), R2 // loads R2 and R3
208 DMB MB_ISH
209
210 MOVW R2, val_lo+4(FP)
211 MOVW R3, val_hi+8(FP)
212 RET
213
214TEXT armStore64<>(SB),NOSPLIT,$0-12
215 // addr is already in R1
216 MOVW val_lo+4(FP), R2
217 MOVW val_hi+8(FP), R3
218
219store64loop:
220 LDREXD (R1), R4 // loads R4 and R5
221
222 DMB MB_ISHST
223
224 STREXD R2, (R1), R0 // stores R2 and R3
225 CMP $0, R0
226 BNE store64loop
227
228 DMB MB_ISH
229 RET
230
231TEXT armAnd8<>(SB),NOSPLIT,$0-5
232 // addr is already in R1
233 MOVB v+4(FP), R2
234
235and8loop:
236 LDREXB (R1), R6
237
238 DMB MB_ISHST
239
240 AND R2, R6
241 STREXB R6, (R1), R0
242 CMP $0, R0
243 BNE and8loop
244
245 DMB MB_ISH
246
247 RET
248
249TEXT armOr8<>(SB),NOSPLIT,$0-5
250 // addr is already in R1
251 MOVB v+4(FP), R2
252
253or8loop:
254 LDREXB (R1), R6
255
256 DMB MB_ISHST
257
258 ORR R2, R6
259 STREXB R6, (R1), R0
260 CMP $0, R0
261 BNE or8loop
262
263 DMB MB_ISH
264
265 RET
266
267TEXT armXchg8<>(SB),NOSPLIT,$0-9
268 // addr is already in R1
269 MOVB v+4(FP), R2
270xchg8loop:
271 LDREXB (R1), R6
272
273 DMB MB_ISHST
274
275 STREXB R2, (R1), R0
276 CMP $0, R0
277 BNE xchg8loop
278
279 DMB MB_ISH
280
281 MOVB R6, ret+8(FP)
282 RET
283
284// The following functions all panic if their address argument isn't
285// 8-byte aligned. Since we're calling back into Go code to do this,
286// we have to cooperate with stack unwinding. In the normal case, the
287// functions tail-call into the appropriate implementation, which
288// means they must not open a frame. Hence, when they go down the
289// panic path, at that point they push the LR to create a real frame
290// (they don't need to pop it because panic won't return; however, we
291// do need to set the SP delta back).
292
293// Check if R1 is 8-byte aligned, panic if not.
294// Clobbers R2.
295#define CHECK_ALIGN \
296 AND.S $7, R1, R2 \
297 BEQ 4(PC) \
298 MOVW.W R14, -4(R13) /* prepare a real frame */ \
299 BL ·panicUnaligned(SB) \
300 ADD $4, R13 /* compensate SP delta */
301
302TEXT ·Cas64(SB),NOSPLIT,$-4-21
303 NO_LOCAL_POINTERS
304 MOVW addr+0(FP), R1
305 CHECK_ALIGN
306
307#ifndef GOARM_7
308 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
309 CMP $1, R11
310 BEQ 2(PC)
311 JMP ·goCas64(SB)
312#endif
313 JMP armCas64<>(SB)
314
315TEXT ·Xadd64(SB),NOSPLIT,$-4-20
316 NO_LOCAL_POINTERS
317 MOVW addr+0(FP), R1
318 CHECK_ALIGN
319
320#ifndef GOARM_7
321 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
322 CMP $1, R11
323 BEQ 2(PC)
324 JMP ·goXadd64(SB)
325#endif
326 JMP armXadd64<>(SB)
327
328TEXT ·Xchg64(SB),NOSPLIT,$-4-20
329 NO_LOCAL_POINTERS
330 MOVW addr+0(FP), R1
331 CHECK_ALIGN
332
333#ifndef GOARM_7
334 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
335 CMP $1, R11
336 BEQ 2(PC)
337 JMP ·goXchg64(SB)
338#endif
339 JMP armXchg64<>(SB)
340
341TEXT ·Load64(SB),NOSPLIT,$-4-12
342 NO_LOCAL_POINTERS
343 MOVW addr+0(FP), R1
344 CHECK_ALIGN
345
346#ifndef GOARM_7
347 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
348 CMP $1, R11
349 BEQ 2(PC)
350 JMP ·goLoad64(SB)
351#endif
352 JMP armLoad64<>(SB)
353
354TEXT ·Store64(SB),NOSPLIT,$-4-12
355 NO_LOCAL_POINTERS
356 MOVW addr+0(FP), R1
357 CHECK_ALIGN
358
359#ifndef GOARM_7
360 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
361 CMP $1, R11
362 BEQ 2(PC)
363 JMP ·goStore64(SB)
364#endif
365 JMP armStore64<>(SB)
366
367TEXT ·And8(SB),NOSPLIT,$-4-5
368 NO_LOCAL_POINTERS
369 MOVW addr+0(FP), R1
370
371// Uses STREXB/LDREXB that is armv6k or later.
372// For simplicity we only enable this on armv7.
373#ifndef GOARM_7
374 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
375 CMP $1, R11
376 BEQ 2(PC)
377 JMP ·goAnd8(SB)
378#endif
379 JMP armAnd8<>(SB)
380
381TEXT ·Or8(SB),NOSPLIT,$-4-5
382 NO_LOCAL_POINTERS
383 MOVW addr+0(FP), R1
384
385// Uses STREXB/LDREXB that is armv6k or later.
386// For simplicity we only enable this on armv7.
387#ifndef GOARM_7
388 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
389 CMP $1, R11
390 BEQ 2(PC)
391 JMP ·goOr8(SB)
392#endif
393 JMP armOr8<>(SB)
394
395TEXT ·Xchg8(SB),NOSPLIT,$-4-9
396 NO_LOCAL_POINTERS
397 MOVW addr+0(FP), R1
398
399 // Uses STREXB/LDREXB that is armv6k or later.
400 // For simplicity we only enable this on armv7.
401#ifndef GOARM_7
402 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
403 CMP $1, R11
404 BEQ 2(PC)
405 JMP ·goXchg8(SB)
406#endif
407 JMP armXchg8<>(SB)
View as plain text