Text file
src/runtime/asm_riscv64.s
Documentation: runtime
1// Copyright 2017 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "funcdata.h"
7#include "textflag.h"
8
9// func rt0_go()
10TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
11 // X2 = stack; A0 = argc; A1 = argv
12 SUB $24, X2
13 MOV A0, 8(X2) // argc
14 MOV A1, 16(X2) // argv
15
16 // create istack out of the given (operating system) stack.
17 // _cgo_init may update stackguard.
18 MOV $runtime·g0(SB), g
19 MOV $(-64*1024), T0
20 ADD T0, X2, T1
21 MOV T1, g_stackguard0(g)
22 MOV T1, g_stackguard1(g)
23 MOV T1, (g_stack+stack_lo)(g)
24 MOV X2, (g_stack+stack_hi)(g)
25
26 // if there is a _cgo_init, call it using the gcc ABI.
27 MOV _cgo_init(SB), T0
28 BEQ T0, ZERO, nocgo
29
30 MOV ZERO, A3 // arg 3: not used
31 MOV ZERO, A2 // arg 2: not used
32 MOV $setg_gcc<>(SB), A1 // arg 1: setg
33 MOV g, A0 // arg 0: G
34 JALR RA, T0
35
36nocgo:
37 // update stackguard after _cgo_init
38 MOV (g_stack+stack_lo)(g), T0
39 ADD $const_stackGuard, T0
40 MOV T0, g_stackguard0(g)
41 MOV T0, g_stackguard1(g)
42
43 // set the per-goroutine and per-mach "registers"
44 MOV $runtime·m0(SB), T0
45
46 // save m->g0 = g0
47 MOV g, m_g0(T0)
48 // save m0 to g0->m
49 MOV T0, g_m(g)
50
51 CALL runtime·check(SB)
52
53 // args are already prepared
54 CALL runtime·args(SB)
55 CALL runtime·osinit(SB)
56 CALL runtime·schedinit(SB)
57
58 // create a new goroutine to start program
59 MOV $runtime·mainPC(SB), T0 // entry
60 SUB $16, X2
61 MOV T0, 8(X2)
62 MOV ZERO, 0(X2)
63 CALL runtime·newproc(SB)
64 ADD $16, X2
65
66 // start this M
67 CALL runtime·mstart(SB)
68
69 WORD $0 // crash if reached
70 RET
71
72TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
73 CALL runtime·mstart0(SB)
74 RET // not reached
75
76// void setg_gcc(G*); set g called from gcc with g in A0
77TEXT setg_gcc<>(SB),NOSPLIT,$0-0
78 MOV A0, g
79 CALL runtime·save_g(SB)
80 RET
81
82// func cputicks() int64
83TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
84 // RDTIME to emulate cpu ticks
85 // RDCYCLE reads counter that is per HART(core) based
86 // according to the riscv manual, see issue 46737
87 RDTIME X10
88 RET
89
90// systemstack_switch is a dummy routine that systemstack leaves at the bottom
91// of the G stack. We need to distinguish the routine that
92// lives at the bottom of the G stack from the one that lives
93// at the top of the system stack because the one at the top of
94// the system stack terminates the stack walk (see topofstack()).
95TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
96 UNDEF
97 JALR RA, ZERO // make sure this function is not leaf
98 RET
99
100// func systemstack(fn func())
101TEXT runtime·systemstack(SB), NOSPLIT, $0-8
102 MOV fn+0(FP), CTXT // CTXT = fn
103 MOV g_m(g), T0 // T0 = m
104
105 MOV m_gsignal(T0), T1 // T1 = gsignal
106 BEQ g, T1, noswitch
107
108 MOV m_g0(T0), T1 // T1 = g0
109 BEQ g, T1, noswitch
110
111 MOV m_curg(T0), T2
112 BEQ g, T2, switch
113
114 // Bad: g is not gsignal, not g0, not curg. What is it?
115 // Hide call from linker nosplit analysis.
116 MOV $runtime·badsystemstack(SB), T1
117 JALR RA, T1
118
119switch:
120 // save our state in g->sched. Pretend to
121 // be systemstack_switch if the G stack is scanned.
122 CALL gosave_systemstack_switch<>(SB)
123
124 // switch to g0
125 MOV T1, g
126 CALL runtime·save_g(SB)
127 MOV (g_sched+gobuf_sp)(g), T0
128 MOV T0, X2
129
130 // call target function
131 MOV 0(CTXT), T1 // code pointer
132 JALR RA, T1
133
134 // switch back to g
135 MOV g_m(g), T0
136 MOV m_curg(T0), g
137 CALL runtime·save_g(SB)
138 MOV (g_sched+gobuf_sp)(g), X2
139 MOV ZERO, (g_sched+gobuf_sp)(g)
140 RET
141
142noswitch:
143 // already on m stack, just call directly
144 // Using a tail call here cleans up tracebacks since we won't stop
145 // at an intermediate systemstack.
146 MOV 0(CTXT), T1 // code pointer
147 ADD $8, X2
148 JMP (T1)
149
150// func switchToCrashStack0(fn func())
151TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
152 MOV X10, CTXT // context register
153 MOV g_m(g), X11 // curm
154
155 // set g to gcrash
156 MOV $runtime·gcrash(SB), g // g = &gcrash
157 CALL runtime·save_g(SB) // clobbers X31
158 MOV X11, g_m(g) // g.m = curm
159 MOV g, m_g0(X11) // curm.g0 = g
160
161 // switch to crashstack
162 MOV (g_stack+stack_hi)(g), X11
163 SUB $(4*8), X11
164 MOV X11, X2
165
166 // call target function
167 MOV 0(CTXT), X10
168 JALR X1, X10
169
170 // should never return
171 CALL runtime·abort(SB)
172 UNDEF
173
174/*
175 * support for morestack
176 */
177
178// Called during function prolog when more stack is needed.
179// Called with return address (i.e. caller's PC) in X5 (aka T0),
180// and the LR register contains the caller's LR.
181//
182// The traceback routines see morestack on a g0 as being
183// the top of a stack (for example, morestack calling newstack
184// calling the scheduler calling newm calling gc), so we must
185// record an argument size. For that purpose, it has no arguments.
186
187// func morestack()
188TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
189 // Called from f.
190 // Set g->sched to context in f.
191 MOV X2, (g_sched+gobuf_sp)(g)
192 MOV T0, (g_sched+gobuf_pc)(g)
193 MOV RA, (g_sched+gobuf_lr)(g)
194 MOV CTXT, (g_sched+gobuf_ctxt)(g)
195
196 // Cannot grow scheduler stack (m->g0).
197 MOV g_m(g), A0
198 MOV m_g0(A0), A1
199 BNE g, A1, 3(PC)
200 CALL runtime·badmorestackg0(SB)
201 CALL runtime·abort(SB)
202
203 // Cannot grow signal stack (m->gsignal).
204 MOV m_gsignal(A0), A1
205 BNE g, A1, 3(PC)
206 CALL runtime·badmorestackgsignal(SB)
207 CALL runtime·abort(SB)
208
209 // Called from f.
210 // Set m->morebuf to f's caller.
211 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
212 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
213 MOV g, (m_morebuf+gobuf_g)(A0)
214
215 // Call newstack on m->g0's stack.
216 MOV m_g0(A0), g
217 CALL runtime·save_g(SB)
218 MOV (g_sched+gobuf_sp)(g), X2
219 // Create a stack frame on g0 to call newstack.
220 MOV ZERO, -8(X2) // Zero saved LR in frame
221 SUB $8, X2
222 CALL runtime·newstack(SB)
223
224 // Not reached, but make sure the return PC from the call to newstack
225 // is still in this function, and not the beginning of the next.
226 UNDEF
227
228// func morestack_noctxt()
229TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
230 // Force SPWRITE. This function doesn't actually write SP,
231 // but it is called with a special calling convention where
232 // the caller doesn't save LR on stack but passes it as a
233 // register, and the unwinder currently doesn't understand.
234 // Make it SPWRITE to stop unwinding. (See issue 54332)
235 MOV X2, X2
236
237 MOV ZERO, CTXT
238 JMP runtime·morestack(SB)
239
240// AES hashing not implemented for riscv64
241TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
242 JMP runtime·memhashFallback<ABIInternal>(SB)
243TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
244 JMP runtime·strhashFallback<ABIInternal>(SB)
245TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
246 JMP runtime·memhash32Fallback<ABIInternal>(SB)
247TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
248 JMP runtime·memhash64Fallback<ABIInternal>(SB)
249
250// func return0()
251TEXT runtime·return0(SB), NOSPLIT, $0
252 MOV $0, A0
253 RET
254
255// restore state from Gobuf; longjmp
256
257// func gogo(buf *gobuf)
258TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
259 MOV buf+0(FP), T0
260 MOV gobuf_g(T0), T1
261 MOV 0(T1), ZERO // make sure g != nil
262 JMP gogo<>(SB)
263
264TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
265 MOV T1, g
266 CALL runtime·save_g(SB)
267
268 MOV gobuf_sp(T0), X2
269 MOV gobuf_lr(T0), RA
270 MOV gobuf_ret(T0), A0
271 MOV gobuf_ctxt(T0), CTXT
272 MOV ZERO, gobuf_sp(T0)
273 MOV ZERO, gobuf_ret(T0)
274 MOV ZERO, gobuf_lr(T0)
275 MOV ZERO, gobuf_ctxt(T0)
276 MOV gobuf_pc(T0), T0
277 JALR ZERO, T0
278
279// func procyield(cycles uint32)
280TEXT runtime·procyield(SB),NOSPLIT,$0-0
281 RET
282
283// Switch to m->g0's stack, call fn(g).
284// Fn must never return. It should gogo(&g->sched)
285// to keep running g.
286
287// func mcall(fn func(*g))
288TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
289 MOV X10, CTXT
290
291 // Save caller state in g->sched
292 MOV X2, (g_sched+gobuf_sp)(g)
293 MOV RA, (g_sched+gobuf_pc)(g)
294 MOV ZERO, (g_sched+gobuf_lr)(g)
295
296 // Switch to m->g0 & its stack, call fn.
297 MOV g, X10
298 MOV g_m(g), T1
299 MOV m_g0(T1), g
300 CALL runtime·save_g(SB)
301 BNE g, X10, 2(PC)
302 JMP runtime·badmcall(SB)
303 MOV 0(CTXT), T1 // code pointer
304 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
305 // we don't need special macro for regabi since arg0(X10) = g
306 SUB $16, X2
307 MOV X10, 8(X2) // setup g
308 MOV ZERO, 0(X2) // clear return address
309 JALR RA, T1
310 JMP runtime·badmcall2(SB)
311
312// Save state of caller into g->sched,
313// but using fake PC from systemstack_switch.
314// Must only be called from functions with no locals ($0)
315// or else unwinding from systemstack_switch is incorrect.
316// Smashes X31.
317TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
318 MOV $runtime·systemstack_switch(SB), X31
319 ADD $8, X31 // get past prologue
320 MOV X31, (g_sched+gobuf_pc)(g)
321 MOV X2, (g_sched+gobuf_sp)(g)
322 MOV ZERO, (g_sched+gobuf_lr)(g)
323 MOV ZERO, (g_sched+gobuf_ret)(g)
324 // Assert ctxt is zero. See func save.
325 MOV (g_sched+gobuf_ctxt)(g), X31
326 BEQ ZERO, X31, 2(PC)
327 CALL runtime·abort(SB)
328 RET
329
330// func asmcgocall_no_g(fn, arg unsafe.Pointer)
331// Call fn(arg) aligned appropriately for the gcc ABI.
332// Called on a system stack, and there may be no g yet (during needm).
333TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
334 MOV fn+0(FP), X5
335 MOV arg+8(FP), X10
336 JALR RA, (X5)
337 RET
338
339// func asmcgocall(fn, arg unsafe.Pointer) int32
340// Call fn(arg) on the scheduler stack,
341// aligned appropriately for the gcc ABI.
342// See cgocall.go for more details.
343TEXT ·asmcgocall(SB),NOSPLIT,$0-20
344 MOV fn+0(FP), X5
345 MOV arg+8(FP), X10
346
347 MOV X2, X8 // save original stack pointer
348 MOV g, X9
349
350 // Figure out if we need to switch to m->g0 stack.
351 // We get called to create new OS threads too, and those
352 // come in on the m->g0 stack already. Or we might already
353 // be on the m->gsignal stack.
354 MOV g_m(g), X6
355 MOV m_gsignal(X6), X7
356 BEQ X7, g, g0
357 MOV m_g0(X6), X7
358 BEQ X7, g, g0
359
360 CALL gosave_systemstack_switch<>(SB)
361 MOV X7, g
362 CALL runtime·save_g(SB)
363 MOV (g_sched+gobuf_sp)(g), X2
364
365 // Now on a scheduling stack (a pthread-created stack).
366g0:
367 // Save room for two of our pointers.
368 SUB $16, X2
369 MOV X9, 0(X2) // save old g on stack
370 MOV (g_stack+stack_hi)(X9), X9
371 SUB X8, X9, X8
372 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
373
374 JALR RA, (X5)
375
376 // Restore g, stack pointer. X10 is return value.
377 MOV 0(X2), g
378 CALL runtime·save_g(SB)
379 MOV (g_stack+stack_hi)(g), X5
380 MOV 8(X2), X6
381 SUB X6, X5, X6
382 MOV X6, X2
383
384 MOVW X10, ret+16(FP)
385 RET
386
387// func asminit()
388TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
389 RET
390
391// reflectcall: call a function with the given argument list
392// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
393// we don't have variable-sized frames, so we use a small number
394// of constant-sized-frame functions to encode a few bits of size in the pc.
395// Caution: ugly multiline assembly macros in your future!
396
397#define DISPATCH(NAME,MAXSIZE) \
398 MOV $MAXSIZE, T1 \
399 BLTU T1, T0, 3(PC) \
400 MOV $NAME(SB), T2; \
401 JALR ZERO, T2
402// Note: can't just "BR NAME(SB)" - bad inlining results.
403
404// func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
405TEXT reflect·call(SB), NOSPLIT, $0-0
406 JMP ·reflectcall(SB)
407
408// func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
409TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
410 MOVWU frameSize+32(FP), T0
411 DISPATCH(runtime·call16, 16)
412 DISPATCH(runtime·call32, 32)
413 DISPATCH(runtime·call64, 64)
414 DISPATCH(runtime·call128, 128)
415 DISPATCH(runtime·call256, 256)
416 DISPATCH(runtime·call512, 512)
417 DISPATCH(runtime·call1024, 1024)
418 DISPATCH(runtime·call2048, 2048)
419 DISPATCH(runtime·call4096, 4096)
420 DISPATCH(runtime·call8192, 8192)
421 DISPATCH(runtime·call16384, 16384)
422 DISPATCH(runtime·call32768, 32768)
423 DISPATCH(runtime·call65536, 65536)
424 DISPATCH(runtime·call131072, 131072)
425 DISPATCH(runtime·call262144, 262144)
426 DISPATCH(runtime·call524288, 524288)
427 DISPATCH(runtime·call1048576, 1048576)
428 DISPATCH(runtime·call2097152, 2097152)
429 DISPATCH(runtime·call4194304, 4194304)
430 DISPATCH(runtime·call8388608, 8388608)
431 DISPATCH(runtime·call16777216, 16777216)
432 DISPATCH(runtime·call33554432, 33554432)
433 DISPATCH(runtime·call67108864, 67108864)
434 DISPATCH(runtime·call134217728, 134217728)
435 DISPATCH(runtime·call268435456, 268435456)
436 DISPATCH(runtime·call536870912, 536870912)
437 DISPATCH(runtime·call1073741824, 1073741824)
438 MOV $runtime·badreflectcall(SB), T2
439 JALR ZERO, T2
440
441#define CALLFN(NAME,MAXSIZE) \
442TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
443 NO_LOCAL_POINTERS; \
444 /* copy arguments to stack */ \
445 MOV stackArgs+16(FP), A1; \
446 MOVWU stackArgsSize+24(FP), A2; \
447 MOV X2, A3; \
448 ADD $8, A3; \
449 ADD A3, A2; \
450 BEQ A3, A2, 6(PC); \
451 MOVBU (A1), A4; \
452 ADD $1, A1; \
453 MOVB A4, (A3); \
454 ADD $1, A3; \
455 JMP -5(PC); \
456 /* set up argument registers */ \
457 MOV regArgs+40(FP), X25; \
458 CALL ·unspillArgs(SB); \
459 /* call function */ \
460 MOV f+8(FP), CTXT; \
461 MOV (CTXT), X25; \
462 PCDATA $PCDATA_StackMapIndex, $0; \
463 JALR RA, X25; \
464 /* copy return values back */ \
465 MOV regArgs+40(FP), X25; \
466 CALL ·spillArgs(SB); \
467 MOV stackArgsType+0(FP), A5; \
468 MOV stackArgs+16(FP), A1; \
469 MOVWU stackArgsSize+24(FP), A2; \
470 MOVWU stackRetOffset+28(FP), A4; \
471 ADD $8, X2, A3; \
472 ADD A4, A3; \
473 ADD A4, A1; \
474 SUB A4, A2; \
475 CALL callRet<>(SB); \
476 RET
477
478// callRet copies return values back at the end of call*. This is a
479// separate function so it can allocate stack space for the arguments
480// to reflectcallmove. It does not follow the Go ABI; it expects its
481// arguments in registers.
482TEXT callRet<>(SB), NOSPLIT, $40-0
483 NO_LOCAL_POINTERS
484 MOV A5, 8(X2)
485 MOV A1, 16(X2)
486 MOV A3, 24(X2)
487 MOV A2, 32(X2)
488 MOV X25, 40(X2)
489 CALL runtime·reflectcallmove(SB)
490 RET
491
492CALLFN(·call16, 16)
493CALLFN(·call32, 32)
494CALLFN(·call64, 64)
495CALLFN(·call128, 128)
496CALLFN(·call256, 256)
497CALLFN(·call512, 512)
498CALLFN(·call1024, 1024)
499CALLFN(·call2048, 2048)
500CALLFN(·call4096, 4096)
501CALLFN(·call8192, 8192)
502CALLFN(·call16384, 16384)
503CALLFN(·call32768, 32768)
504CALLFN(·call65536, 65536)
505CALLFN(·call131072, 131072)
506CALLFN(·call262144, 262144)
507CALLFN(·call524288, 524288)
508CALLFN(·call1048576, 1048576)
509CALLFN(·call2097152, 2097152)
510CALLFN(·call4194304, 4194304)
511CALLFN(·call8388608, 8388608)
512CALLFN(·call16777216, 16777216)
513CALLFN(·call33554432, 33554432)
514CALLFN(·call67108864, 67108864)
515CALLFN(·call134217728, 134217728)
516CALLFN(·call268435456, 268435456)
517CALLFN(·call536870912, 536870912)
518CALLFN(·call1073741824, 1073741824)
519
520// Called from cgo wrappers, this function returns g->m->curg.stack.hi.
521// Must obey the gcc calling convention.
522TEXT _cgo_topofstack(SB),NOSPLIT,$8
523 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
524 // X27 is callee-save in the gcc calling convention, so save it.
525 MOV g, savedX27-8(SP)
526
527 CALL runtime·load_g(SB)
528 MOV g_m(g), X5
529 MOV m_curg(X5), X5
530 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
531
532 MOV savedX27-8(SP), g
533 RET
534
535// func goexit(neverCallThisFunction)
536// The top-most function running on a goroutine
537// returns to goexit+PCQuantum.
538TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
539 MOV ZERO, ZERO // NOP
540 JMP runtime·goexit1(SB) // does not return
541 // traceback from goexit1 must hit code range of goexit
542 MOV ZERO, ZERO // NOP
543
544// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
545// See cgocall.go for more details.
546TEXT ·cgocallback(SB),NOSPLIT,$24-24
547 NO_LOCAL_POINTERS
548
549 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
550 // It is used to dropm while thread is exiting.
551 MOV fn+0(FP), X7
552 BNE ZERO, X7, loadg
553 // Restore the g from frame.
554 MOV frame+8(FP), g
555 JMP dropm
556
557loadg:
558 // Load m and g from thread-local storage.
559 MOVBU runtime·iscgo(SB), X5
560 BEQ ZERO, X5, nocgo
561 CALL runtime·load_g(SB)
562nocgo:
563
564 // If g is nil, Go did not create the current thread,
565 // or if this thread never called into Go on pthread platforms.
566 // Call needm to obtain one for temporary use.
567 // In this case, we're running on the thread stack, so there's
568 // lots of space, but the linker doesn't know. Hide the call from
569 // the linker analysis by using an indirect call.
570 BEQ ZERO, g, needm
571
572 MOV g_m(g), X5
573 MOV X5, savedm-8(SP)
574 JMP havem
575
576needm:
577 MOV g, savedm-8(SP) // g is zero, so is m.
578 MOV $runtime·needAndBindM(SB), X6
579 JALR RA, X6
580
581 // Set m->sched.sp = SP, so that if a panic happens
582 // during the function we are about to execute, it will
583 // have a valid SP to run on the g0 stack.
584 // The next few lines (after the havem label)
585 // will save this SP onto the stack and then write
586 // the same SP back to m->sched.sp. That seems redundant,
587 // but if an unrecovered panic happens, unwindm will
588 // restore the g->sched.sp from the stack location
589 // and then systemstack will try to use it. If we don't set it here,
590 // that restored SP will be uninitialized (typically 0) and
591 // will not be usable.
592 MOV g_m(g), X5
593 MOV m_g0(X5), X6
594 MOV X2, (g_sched+gobuf_sp)(X6)
595
596havem:
597 // Now there's a valid m, and we're running on its m->g0.
598 // Save current m->g0->sched.sp on stack and then set it to SP.
599 // Save current sp in m->g0->sched.sp in preparation for
600 // switch back to m->curg stack.
601 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
602 MOV m_g0(X5), X6
603 MOV (g_sched+gobuf_sp)(X6), X7
604 MOV X7, savedsp-24(SP) // must match frame size
605 MOV X2, (g_sched+gobuf_sp)(X6)
606
607 // Switch to m->curg stack and call runtime.cgocallbackg.
608 // Because we are taking over the execution of m->curg
609 // but *not* resuming what had been running, we need to
610 // save that information (m->curg->sched) so we can restore it.
611 // We can restore m->curg->sched.sp easily, because calling
612 // runtime.cgocallbackg leaves SP unchanged upon return.
613 // To save m->curg->sched.pc, we push it onto the curg stack and
614 // open a frame the same size as cgocallback's g0 frame.
615 // Once we switch to the curg stack, the pushed PC will appear
616 // to be the return PC of cgocallback, so that the traceback
617 // will seamlessly trace back into the earlier calls.
618 MOV m_curg(X5), g
619 CALL runtime·save_g(SB)
620 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
621 MOV (g_sched+gobuf_pc)(g), X7
622 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
623 // Gather our arguments into registers.
624 MOV fn+0(FP), X7
625 MOV frame+8(FP), X8
626 MOV ctxt+16(FP), X9
627 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
628 MOV X7, 8(X2)
629 MOV X8, 16(X2)
630 MOV X9, 24(X2)
631 CALL runtime·cgocallbackg(SB)
632
633 // Restore g->sched (== m->curg->sched) from saved values.
634 MOV 0(X2), X7
635 MOV X7, (g_sched+gobuf_pc)(g)
636 MOV $(24+8)(X2), X6 // must match frame size
637 MOV X6, (g_sched+gobuf_sp)(g)
638
639 // Switch back to m->g0's stack and restore m->g0->sched.sp.
640 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
641 // so we do not have to restore it.)
642 MOV g_m(g), X5
643 MOV m_g0(X5), g
644 CALL runtime·save_g(SB)
645 MOV (g_sched+gobuf_sp)(g), X2
646 MOV savedsp-24(SP), X6 // must match frame size
647 MOV X6, (g_sched+gobuf_sp)(g)
648
649 // If the m on entry was nil, we called needm above to borrow an m,
650 // 1. for the duration of the call on non-pthread platforms,
651 // 2. or the duration of the C thread alive on pthread platforms.
652 // If the m on entry wasn't nil,
653 // 1. the thread might be a Go thread,
654 // 2. or it wasn't the first call from a C thread on pthread platforms,
655 // since then we skip dropm to reuse the m in the first call.
656 MOV savedm-8(SP), X5
657 BNE ZERO, X5, droppedm
658
659 // Skip dropm to reuse it in the next call, when a pthread key has been created.
660 MOV _cgo_pthread_key_created(SB), X5
661 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
662 BEQ ZERO, X5, dropm
663 MOV (X5), X5
664 BNE ZERO, X5, droppedm
665
666dropm:
667 MOV $runtime·dropm(SB), X6
668 JALR RA, X6
669droppedm:
670
671 // Done!
672 RET
673
674TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
675 EBREAK
676 RET
677
678TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
679 EBREAK
680 RET
681
682// void setg(G*); set g. for use by needm.
683TEXT runtime·setg(SB), NOSPLIT, $0-8
684 MOV gg+0(FP), g
685 // This only happens if iscgo, so jump straight to save_g
686 CALL runtime·save_g(SB)
687 RET
688
689TEXT ·checkASM(SB),NOSPLIT,$0-1
690 MOV $1, T0
691 MOV T0, ret+0(FP)
692 RET
693
694// spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
695TEXT ·spillArgs(SB),NOSPLIT,$0-0
696 MOV X10, (0*8)(X25)
697 MOV X11, (1*8)(X25)
698 MOV X12, (2*8)(X25)
699 MOV X13, (3*8)(X25)
700 MOV X14, (4*8)(X25)
701 MOV X15, (5*8)(X25)
702 MOV X16, (6*8)(X25)
703 MOV X17, (7*8)(X25)
704 MOV X8, (8*8)(X25)
705 MOV X9, (9*8)(X25)
706 MOV X18, (10*8)(X25)
707 MOV X19, (11*8)(X25)
708 MOV X20, (12*8)(X25)
709 MOV X21, (13*8)(X25)
710 MOV X22, (14*8)(X25)
711 MOV X23, (15*8)(X25)
712 MOVD F10, (16*8)(X25)
713 MOVD F11, (17*8)(X25)
714 MOVD F12, (18*8)(X25)
715 MOVD F13, (19*8)(X25)
716 MOVD F14, (20*8)(X25)
717 MOVD F15, (21*8)(X25)
718 MOVD F16, (22*8)(X25)
719 MOVD F17, (23*8)(X25)
720 MOVD F8, (24*8)(X25)
721 MOVD F9, (25*8)(X25)
722 MOVD F18, (26*8)(X25)
723 MOVD F19, (27*8)(X25)
724 MOVD F20, (28*8)(X25)
725 MOVD F21, (29*8)(X25)
726 MOVD F22, (30*8)(X25)
727 MOVD F23, (31*8)(X25)
728 RET
729
730// unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
731TEXT ·unspillArgs(SB),NOSPLIT,$0-0
732 MOV (0*8)(X25), X10
733 MOV (1*8)(X25), X11
734 MOV (2*8)(X25), X12
735 MOV (3*8)(X25), X13
736 MOV (4*8)(X25), X14
737 MOV (5*8)(X25), X15
738 MOV (6*8)(X25), X16
739 MOV (7*8)(X25), X17
740 MOV (8*8)(X25), X8
741 MOV (9*8)(X25), X9
742 MOV (10*8)(X25), X18
743 MOV (11*8)(X25), X19
744 MOV (12*8)(X25), X20
745 MOV (13*8)(X25), X21
746 MOV (14*8)(X25), X22
747 MOV (15*8)(X25), X23
748 MOVD (16*8)(X25), F10
749 MOVD (17*8)(X25), F11
750 MOVD (18*8)(X25), F12
751 MOVD (19*8)(X25), F13
752 MOVD (20*8)(X25), F14
753 MOVD (21*8)(X25), F15
754 MOVD (22*8)(X25), F16
755 MOVD (23*8)(X25), F17
756 MOVD (24*8)(X25), F8
757 MOVD (25*8)(X25), F9
758 MOVD (26*8)(X25), F18
759 MOVD (27*8)(X25), F19
760 MOVD (28*8)(X25), F20
761 MOVD (29*8)(X25), F21
762 MOVD (30*8)(X25), F22
763 MOVD (31*8)(X25), F23
764 RET
765
766// gcWriteBarrier informs the GC about heap pointer writes.
767//
768// gcWriteBarrier does NOT follow the Go ABI. It accepts the
769// number of bytes of buffer needed in X24, and returns a pointer
770// to the buffer space in X24.
771// It clobbers X31 aka T6 (the linker temp register - REG_TMP).
772// The act of CALLing gcWriteBarrier will clobber RA (LR).
773// It does not clobber any other general-purpose registers,
774// but may clobber others (e.g., floating point registers).
775TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
776 // Save the registers clobbered by the fast path.
777 MOV A0, 24*8(X2)
778 MOV A1, 25*8(X2)
779retry:
780 MOV g_m(g), A0
781 MOV m_p(A0), A0
782 MOV (p_wbBuf+wbBuf_next)(A0), A1
783 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
784 // Increment wbBuf.next position.
785 ADD X24, A1
786 // Is the buffer full?
787 BLTU T6, A1, flush
788 // Commit to the larger buffer.
789 MOV A1, (p_wbBuf+wbBuf_next)(A0)
790 // Make the return value (the original next position)
791 SUB X24, A1, X24
792 // Restore registers.
793 MOV 24*8(X2), A0
794 MOV 25*8(X2), A1
795 RET
796
797flush:
798 // Save all general purpose registers since these could be
799 // clobbered by wbBufFlush and were not saved by the caller.
800 MOV T0, 1*8(X2)
801 MOV T1, 2*8(X2)
802 // X0 is zero register
803 // X1 is LR, saved by prologue
804 // X2 is SP
805 // X3 is GP
806 // X4 is TP
807 MOV X7, 3*8(X2)
808 MOV X8, 4*8(X2)
809 MOV X9, 5*8(X2)
810 // X10 already saved (A0)
811 // X11 already saved (A1)
812 MOV X12, 6*8(X2)
813 MOV X13, 7*8(X2)
814 MOV X14, 8*8(X2)
815 MOV X15, 9*8(X2)
816 MOV X16, 10*8(X2)
817 MOV X17, 11*8(X2)
818 MOV X18, 12*8(X2)
819 MOV X19, 13*8(X2)
820 MOV X20, 14*8(X2)
821 MOV X21, 15*8(X2)
822 MOV X22, 16*8(X2)
823 MOV X23, 17*8(X2)
824 MOV X24, 18*8(X2)
825 MOV X25, 19*8(X2)
826 MOV X26, 20*8(X2)
827 // X27 is g.
828 MOV X28, 21*8(X2)
829 MOV X29, 22*8(X2)
830 MOV X30, 23*8(X2)
831 // X31 is tmp register.
832
833 CALL runtime·wbBufFlush(SB)
834
835 MOV 1*8(X2), T0
836 MOV 2*8(X2), T1
837 MOV 3*8(X2), X7
838 MOV 4*8(X2), X8
839 MOV 5*8(X2), X9
840 MOV 6*8(X2), X12
841 MOV 7*8(X2), X13
842 MOV 8*8(X2), X14
843 MOV 9*8(X2), X15
844 MOV 10*8(X2), X16
845 MOV 11*8(X2), X17
846 MOV 12*8(X2), X18
847 MOV 13*8(X2), X19
848 MOV 14*8(X2), X20
849 MOV 15*8(X2), X21
850 MOV 16*8(X2), X22
851 MOV 17*8(X2), X23
852 MOV 18*8(X2), X24
853 MOV 19*8(X2), X25
854 MOV 20*8(X2), X26
855 MOV 21*8(X2), X28
856 MOV 22*8(X2), X29
857 MOV 23*8(X2), X30
858
859 JMP retry
860
861TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
862 MOV $8, X24
863 JMP gcWriteBarrier<>(SB)
864TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
865 MOV $16, X24
866 JMP gcWriteBarrier<>(SB)
867TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
868 MOV $24, X24
869 JMP gcWriteBarrier<>(SB)
870TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
871 MOV $32, X24
872 JMP gcWriteBarrier<>(SB)
873TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
874 MOV $40, X24
875 JMP gcWriteBarrier<>(SB)
876TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
877 MOV $48, X24
878 JMP gcWriteBarrier<>(SB)
879TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
880 MOV $56, X24
881 JMP gcWriteBarrier<>(SB)
882TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
883 MOV $64, X24
884 JMP gcWriteBarrier<>(SB)
885
886// Note: these functions use a special calling convention to save generated code space.
887// Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those
888// arguments are allocated in the caller's stack frame.
889// These stubs write the args into that stack space and then tail call to the
890// corresponding runtime handler.
891// The tail call makes these stubs disappear in backtraces.
892TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
893 MOV T0, X10
894 MOV T1, X11
895 JMP runtime·goPanicIndex<ABIInternal>(SB)
896TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
897 MOV T0, X10
898 MOV T1, X11
899 JMP runtime·goPanicIndexU<ABIInternal>(SB)
900TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
901 MOV T1, X10
902 MOV T2, X11
903 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
904TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
905 MOV T1, X10
906 MOV T2, X11
907 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
908TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
909 MOV T1, X10
910 MOV T2, X11
911 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
912TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
913 MOV T1, X10
914 MOV T2, X11
915 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
916TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
917 MOV T0, X10
918 MOV T1, X11
919 JMP runtime·goPanicSliceB<ABIInternal>(SB)
920TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
921 MOV T0, X10
922 MOV T1, X11
923 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
924TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
925 MOV T2, X10
926 MOV T3, X11
927 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
928TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
929 MOV T2, X10
930 MOV T3, X11
931 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
932TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
933 MOV T2, X10
934 MOV T3, X11
935 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
936TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
937 MOV T2, X10
938 MOV T3, X11
939 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
940TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
941 MOV T1, X10
942 MOV T2, X11
943 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
944TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
945 MOV T1, X10
946 MOV T2, X11
947 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
948TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
949 MOV T0, X10
950 MOV T1, X11
951 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
952TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
953 MOV T0, X10
954 MOV T1, X11
955 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
956TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
957 MOV T2, X10
958 MOV T3, X11
959 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
960
961DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
962GLOBL runtime·mainPC(SB),RODATA,$8
View as plain text