-
Notifications
You must be signed in to change notification settings - Fork 755
/
syscall_asm_amd64.s
1308 lines (1146 loc) · 36.4 KB
/
syscall_asm_amd64.s
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
*/
#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/machbrand.h>
#if defined(__lint)
#include <sys/types.h>
#include <sys/thread.h>
#include <sys/systm.h>
#else /* __lint */
#include <sys/segments.h>
#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/model.h>
#include <sys/panic.h>
#if defined(__xpv)
#include <sys/hypervisor.h>
#endif
#include "assym.h"
#endif /* __lint */
/*
* We implement five flavours of system call entry points
*
* - syscall/sysretq (amd64 generic)
* - syscall/sysretl (i386 plus SYSC bit)
* - sysenter/sysexit (i386 plus SEP bit)
* - int/iret (i386 generic)
* - lcall/iret (i386 generic)
*
* The current libc included in Solaris uses int/iret as the base unoptimized
* kernel entry method. Older libc implementations and legacy binaries may use
* the lcall call gate, so it must continue to be supported.
*
* System calls that use an lcall call gate are processed in trap() via a
* segment-not-present trap, i.e. lcalls are extremely slow(!).
*
* The basic pattern used in the 32-bit SYSC handler at this point in time is
* to have the bare minimum of assembler, and get to the C handlers as
* quickly as possible.
*
* The 64-bit handler is much closer to the sparcv9 handler; that's
* because of passing arguments in registers. The 32-bit world still
* passes arguments on the stack -- that makes that handler substantially
* more complex.
*
* The two handlers share a few code fragments which are broken
* out into preprocessor macros below.
*
* XX64 come back and speed all this up later. The 32-bit stuff looks
* especially easy to speed up the argument copying part ..
*
*
* Notes about segment register usage (c.f. the 32-bit kernel)
*
* In the 32-bit kernel, segment registers are dutifully saved and
* restored on all mode transitions because the kernel uses them directly.
* When the processor is running in 64-bit mode, segment registers are
* largely ignored.
*
* %cs and %ss
* controlled by the hardware mechanisms that make mode transitions
*
* The remaining segment registers have to either be pointing at a valid
* descriptor i.e. with the 'present' bit set, or they can NULL descriptors
*
* %ds and %es
* always ignored
*
* %fs and %gs
* fsbase and gsbase are used to control the place they really point at.
* The kernel only depends on %gs, and controls its own gsbase via swapgs
*
* Note that loading segment registers is still costly because the GDT
* lookup still happens (this is because the hardware can't know that we're
* not setting up these segment registers for a 32-bit program). Thus we
* avoid doing this in the syscall path, and defer them to lwp context switch
* handlers, so the register values remain virtualized to the lwp.
*/
#if defined(SYSCALLTRACE)
#define ORL_SYSCALLTRACE(r32) \
orl syscalltrace(%rip), r32
#else
#define ORL_SYSCALLTRACE(r32)
#endif
/*
* In the 32-bit kernel, we do absolutely nothing before getting into the
* brand callback checks. In 64-bit land, we do swapgs and then come here.
* We assume that the %rsp- and %r15-stashing fields in the CPU structure
* are still unused.
*
* Check if a brand_mach_ops callback is defined for the specified callback_id
* type. If so invoke it with the kernel's %gs value loaded and the following
* data on the stack:
*
* stack: --------------------------------------
* 32 | callback pointer |
* | 24 | user (or interrupt) stack pointer |
* | 16 | lwp pointer |
* v 8 | userland return address |
* 0 | callback wrapper return addr |
* --------------------------------------
*
* Since we're pushing the userland return address onto the kernel stack
* we need to get that address without accessing the user's stack (since we
* can't trust that data). There are different ways to get the userland
* return address depending on how the syscall trap was made:
*
* a) For sys_syscall and sys_syscall32 the return address is in %rcx.
* b) For sys_sysenter the return address is in %rdx.
* c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
* the stack pointer points at the state saved when we took the interrupt:
* ------------------------
* | | user's %ss |
* | | user's %esp |
* | | EFLAGS register |
* v | user's %cs |
* | user's %eip |
* ------------------------
*
* The 2nd parameter to the BRAND_CALLBACK macro is either the
* BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
* used to generate the proper code to get the userland return address for
* each syscall entry point.
*
* The interface to the brand callbacks on the 64-bit kernel assumes %r15
* is available as a scratch register within the callback. If the callback
* returns within the kernel then this macro will restore %r15. If the
* callback is going to return directly to userland then it should restore
* %r15 before returning to userland.
*/
#define BRAND_URET_FROM_REG(rip_reg) \
pushq rip_reg /* push the return address */
/*
* The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
* is currently pointing at the user return address (%eip).
*/
#define BRAND_URET_FROM_INTR_STACK() \
movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
pushq (%r15) /* push the return address */
#define BRAND_CALLBACK(callback_id, push_userland_ret) \
movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
subq $16, %rsp /* save space for 2 pointers */ ;\
pushq %r14 /* save %r14 */ ;\
movq %gs:CPU_RTMP_RSP, %r14 ;\
movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
popq %r14 /* restore %r14 */ ;\
movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
pushq %r15 /* push the lwp pointer */ ;\
movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
cmpq $0, %r15 ;\
je 1f ;\
movq %r15, 16(%rsp) /* save the callback pointer */ ;\
push_userland_ret /* push the return address */ ;\
call *24(%rsp) /* call callback */ ;\
1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
#define MSTATE_TRANSITION(from, to) \
movl $from, %edi; \
movl $to, %esi; \
call syscall_mstate
/*
* Check to see if a simple (direct) return is possible i.e.
*
* if (t->t_post_sys_ast | syscalltrace |
* lwp->lwp_pcb.pcb_rupdate == 1)
* do full version ;
*
* Preconditions:
* - t is curthread
* Postconditions:
* - condition code NE is set if post-sys is too complex
* - rtmp is zeroed if it isn't (we rely on this!)
* - ltmp is smashed
*/
#define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
movq T_LWP(t), ltmp; \
movzbl PCB_RUPDATE(ltmp), rtmp; \
ORL_SYSCALLTRACE(rtmp); \
orl T_POST_SYS_AST(t), rtmp; \
cmpl $0, rtmp
/*
* Fix up the lwp, thread, and eflags for a successful return
*
* Preconditions:
* - zwreg contains zero
*/
#define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
movb $LWP_USER, LWP_STATE(lwp); \
movw zwreg, T_SYSNUM(t); \
andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
/*
* ASSERT(lwptoregs(lwp) == rp);
*
* This may seem obvious, but very odd things happen if this
* assertion is false
*
* Preconditions:
* (%rsp is ready for normal call sequence)
* Postconditions (if assertion is true):
* %r11 is smashed
*
* ASSERT(rp->r_cs == descnum)
*
* The code selector is written into the regs structure when the
* lwp stack is created. We use this ASSERT to validate that
* the regs structure really matches how we came in.
*
* Preconditions:
* (%rsp is ready for normal call sequence)
* Postconditions (if assertion is true):
* -none-
*
* ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
*
* If this is false, it meant that we returned to userland without
* updating the segment registers as we were supposed to.
*
* Note that we must ensure no interrupts or other traps intervene
* between entering privileged mode and performing the assertion,
* otherwise we may perform a context switch on the thread, which
* will end up setting pcb_rupdate to 1 again.
*/
#if defined(DEBUG)
#if !defined(__lint)
__lwptoregs_msg:
.string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
__codesel_msg:
.string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
__no_rupdate_msg:
.string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
#endif /* !__lint */
#define ASSERT_LWPTOREGS(lwp, rp) \
movq LWP_REGS(lwp), %r11; \
cmpq rp, %r11; \
je 7f; \
leaq __lwptoregs_msg(%rip), %rdi; \
movl $__LINE__, %esi; \
movq lwp, %rdx; \
movq %r11, %rcx; \
movq rp, %r8; \
xorl %eax, %eax; \
call panic; \
7:
#define ASSERT_NO_RUPDATE_PENDING(lwp) \
testb $0x1, PCB_RUPDATE(lwp); \
je 8f; \
movq lwp, %rdx; \
leaq __no_rupdate_msg(%rip), %rdi; \
movl $__LINE__, %esi; \
xorl %eax, %eax; \
call panic; \
8:
#else
#define ASSERT_LWPTOREGS(lwp, rp)
#define ASSERT_NO_RUPDATE_PENDING(lwp)
#endif
/*
* Do the traptrace thing and restore any registers we used
* in situ. Assumes that %rsp is pointing at the base of
* the struct regs, obviously ..
*/
#ifdef TRAPTRACE
#define SYSCALL_TRAPTRACE(ttype) \
TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
movq REGOFF_RAX(%rsp), %rax; \
movq REGOFF_RBX(%rsp), %rbx; \
movq REGOFF_RCX(%rsp), %rcx; \
movq REGOFF_RDX(%rsp), %rdx; \
movl %eax, TTR_SYSNUM(%rdi); \
movq REGOFF_RDI(%rsp), %rdi
#define SYSCALL_TRAPTRACE32(ttype) \
SYSCALL_TRAPTRACE(ttype); \
/* paranoia: clean the top 32-bits of the registers */ \
orl %eax, %eax; \
orl %ebx, %ebx; \
orl %ecx, %ecx; \
orl %edx, %edx; \
orl %edi, %edi
#else /* TRAPTRACE */
#define SYSCALL_TRAPTRACE(ttype)
#define SYSCALL_TRAPTRACE32(ttype)
#endif /* TRAPTRACE */
/*
* The 64-bit libc syscall wrapper does this:
*
* fn(<args>)
* {
* movq %rcx, %r10 -- because syscall smashes %rcx
* movl $CODE, %eax
* syscall
* <error processing>
* }
*
* Thus when we come into the kernel:
*
* %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
* %rax is the syscall number
* %r12-%r15 contain caller state
*
* The syscall instruction arranges that:
*
* %rcx contains the return %rip
* %r11d contains bottom 32-bits of %rflags
* %rflags is masked (as determined by the SFMASK msr)
* %cs is set to UCS_SEL (as determined by the STAR msr)
* %ss is set to UDS_SEL (as determined by the STAR msr)
* %rip is set to sys_syscall (as determined by the LSTAR msr)
*
* Or in other words, we have no registers available at all.
* Only swapgs can save us!
*
* Under the hypervisor, the swapgs has happened already. However, the
* state of the world is very different from that we're familiar with.
*
* In particular, we have a stack structure like that for interrupt
* gates, except that the %cs and %ss registers are modified for reasons
* that are not entirely clear. Critically, the %rcx/%r11 values do
* *not* reflect the usage of those registers under a 'real' syscall[1];
* the stack, therefore, looks like this:
*
* 0x0(rsp) potentially junk %rcx
* 0x8(rsp) potentially junk %r11
* 0x10(rsp) user %rip
* 0x18(rsp) modified %cs
* 0x20(rsp) user %rflags
* 0x28(rsp) user %rsp
* 0x30(rsp) modified %ss
*
*
* and before continuing on, we must load the %rip into %rcx and the
* %rflags into %r11.
*
* [1] They used to, and we relied on it, but this was broken in 3.1.1.
* Sigh.
*/
#if defined(__xpv)
#define XPV_SYSCALL_PROD \
movq 0x10(%rsp), %rcx; \
movq 0x20(%rsp), %r11; \
movq 0x28(%rsp), %rsp
#else
#define XPV_SYSCALL_PROD /* nothing */
#endif
#if defined(__lint)
/*ARGSUSED*/
void
sys_syscall()
{}
void
_allsyscalls()
{}
size_t _allsyscalls_size;
#else /* __lint */
ENTRY_NP2(brand_sys_syscall,_allsyscalls)
SWAPGS /* kernel gsbase */
XPV_SYSCALL_PROD
BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
jmp noprod_sys_syscall
ALTENTRY(sys_syscall)
SWAPGS /* kernel gsbase */
XPV_SYSCALL_PROD
noprod_sys_syscall:
movq %r15, %gs:CPU_RTMP_R15
movq %rsp, %gs:CPU_RTMP_RSP
movq %gs:CPU_THREAD, %r15
movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
ASSERT_UPCALL_MASK_IS_SET
movl $UCS_SEL, REGOFF_CS(%rsp)
movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
movl $UDS_SEL, REGOFF_SS(%rsp)
movl %eax, %eax /* wrapper: sysc# -> %eax */
movq %rdi, REGOFF_RDI(%rsp)
movq %rsi, REGOFF_RSI(%rsp)
movq %rdx, REGOFF_RDX(%rsp)
movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
movq %r10, %rcx /* arg[3] for direct calls */
movq %r8, REGOFF_R8(%rsp)
movq %r9, REGOFF_R9(%rsp)
movq %rax, REGOFF_RAX(%rsp)
movq %rbx, REGOFF_RBX(%rsp)
movq %rbp, REGOFF_RBP(%rsp)
movq %r10, REGOFF_R10(%rsp)
movq %gs:CPU_RTMP_RSP, %r11
movq %r11, REGOFF_RSP(%rsp)
movq %r12, REGOFF_R12(%rsp)
movq %r13, REGOFF_R13(%rsp)
movq %r14, REGOFF_R14(%rsp)
movq %gs:CPU_RTMP_R15, %r10
movq %r10, REGOFF_R15(%rsp)
movq $0, REGOFF_SAVFP(%rsp)
movq $0, REGOFF_SAVPC(%rsp)
/*
* Copy these registers here in case we end up stopped with
* someone (like, say, /proc) messing with our register state.
* We don't -restore- them unless we have to in update_sregs.
*
* Since userland -can't- change fsbase or gsbase directly,
* and capturing them involves two serializing instructions,
* we don't bother to capture them here.
*/
xorl %ebx, %ebx
movw %ds, %bx
movq %rbx, REGOFF_DS(%rsp)
movw %es, %bx
movq %rbx, REGOFF_ES(%rsp)
movw %fs, %bx
movq %rbx, REGOFF_FS(%rsp)
movw %gs, %bx
movq %rbx, REGOFF_GS(%rsp)
/*
* Machine state saved in the regs structure on the stack
* First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
* %eax is the syscall number
* %rsp is the thread's stack, %r15 is curthread
* REG_RSP(%rsp) is the user's stack
*/
SYSCALL_TRAPTRACE($TT_SYSC64)
movq %rsp, %rbp
movq T_LWP(%r15), %r14
ASSERT_NO_RUPDATE_PENDING(%r14)
ENABLE_INTR_FLAGS
MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
ASSERT_LWPTOREGS(%r14, %rsp)
movb $LWP_SYS, LWP_STATE(%r14)
incq LWP_RU_SYSC(%r14)
movb $NORMALRETURN, LWP_EOSYS(%r14)
incq %gs:CPU_STATS_SYS_SYSCALL
movw %ax, T_SYSNUM(%r15)
movzbl T_PRE_SYS(%r15), %ebx
ORL_SYSCALLTRACE(%ebx)
testl %ebx, %ebx
jne _syscall_pre
_syscall_invoke:
movq REGOFF_RDI(%rbp), %rdi
movq REGOFF_RSI(%rbp), %rsi
movq REGOFF_RDX(%rbp), %rdx
movq REGOFF_RCX(%rbp), %rcx
movq REGOFF_R8(%rbp), %r8
movq REGOFF_R9(%rbp), %r9
cmpl $NSYSCALL, %eax
jae _syscall_ill
shll $SYSENT_SIZE_SHIFT, %eax
leaq sysent(%rax), %rbx
call *SY_CALLC(%rbx)
movq %rax, %r12
movq %rdx, %r13
/*
* If the handler returns two ints, then we need to split the
* 64-bit return value into two 32-bit values.
*/
testw $SE_32RVAL2, SY_FLAGS(%rbx)
je 5f
movq %r12, %r13
shrq $32, %r13 /* upper 32-bits into %edx */
movl %r12d, %r12d /* lower 32-bits into %eax */
5:
/*
* Optimistically assume that there's no post-syscall
* work to do. (This is to avoid having to call syscall_mstate()
* with interrupts disabled)
*/
MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
/*
* We must protect ourselves from being descheduled here;
* If we were, and we ended up on another cpu, or another
* lwp got in ahead of us, it could change the segment
* registers without us noticing before we return to userland.
*/
CLI(%r14)
CHECK_POSTSYS_NE(%r15, %r14, %ebx)
jne _syscall_post
/*
* We need to protect ourselves against non-canonical return values
* because Intel doesn't check for them on sysret (AMD does). Canonical
* addresses on current amd64 processors only use 48-bits for VAs; an
* address is canonical if all upper bits (47-63) are identical. If we
* find a non-canonical %rip, we opt to go through the full
* _syscall_post path which takes us into an iretq which is not
* susceptible to the same problems sysret is.
*
* We're checking for a canonical address by first doing an arithmetic
* shift. This will fill in the remaining bits with the value of bit 63.
* If the address were canonical, the register would now have either all
* zeroes or all ones in it. Therefore we add one (inducing overflow)
* and compare against 1. A canonical address will either be zero or one
* at this point, hence the use of ja.
*
* At this point, r12 and r13 have the return value so we can't use
* those registers.
*/
movq REGOFF_RIP(%rsp), %rcx
sarq $47, %rcx
incq %rcx
cmpq $1, %rcx
ja _syscall_post
SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
movq %r12, REGOFF_RAX(%rsp)
movq %r13, REGOFF_RDX(%rsp)
/*
* To get back to userland, we need the return %rip in %rcx and
* the return %rfl in %r11d. The sysretq instruction also arranges
* to fix up %cs and %ss; everything else is our responsibility.
*/
movq REGOFF_RDI(%rsp), %rdi
movq REGOFF_RSI(%rsp), %rsi
movq REGOFF_RDX(%rsp), %rdx
/* %rcx used to restore %rip value */
movq REGOFF_R8(%rsp), %r8
movq REGOFF_R9(%rsp), %r9
movq REGOFF_RAX(%rsp), %rax
movq REGOFF_RBX(%rsp), %rbx
movq REGOFF_RBP(%rsp), %rbp
movq REGOFF_R10(%rsp), %r10
/* %r11 used to restore %rfl value */
movq REGOFF_R12(%rsp), %r12
movq REGOFF_R13(%rsp), %r13
movq REGOFF_R14(%rsp), %r14
movq REGOFF_R15(%rsp), %r15
movq REGOFF_RIP(%rsp), %rcx
movl REGOFF_RFL(%rsp), %r11d
#if defined(__xpv)
addq $REGOFF_RIP, %rsp
#else
movq REGOFF_RSP(%rsp), %rsp
#endif
/*
* There can be no instructions between the ALTENTRY below and
* SYSRET or we could end up breaking brand support. See label usage
* in sn1_brand_syscall_callback for an example.
*/
ASSERT_UPCALL_MASK_IS_SET
#if defined(__xpv)
SYSRETQ
ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
/*
* We can only get here after executing a brand syscall
* interposition callback handler and simply need to
* "sysretq" back to userland. On the hypervisor this
* involves the iret hypercall which requires us to construct
* just enough of the stack needed for the hypercall.
* (rip, cs, rflags, rsp, ss).
*/
movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
movq %gs:CPU_THREAD, %r11
movq T_STACK(%r11), %rsp
movq %rcx, REGOFF_RIP(%rsp)
movl $UCS_SEL, REGOFF_CS(%rsp)
movq %gs:CPU_RTMP_RSP, %r11
movq %r11, REGOFF_RSP(%rsp)
pushfq
popq %r11 /* hypercall enables ints */
movq %r11, REGOFF_RFL(%rsp)
movl $UDS_SEL, REGOFF_SS(%rsp)
addq $REGOFF_RIP, %rsp
/*
* XXPV: see comment in SYSRETQ definition for future optimization
* we could take.
*/
ASSERT_UPCALL_MASK_IS_SET
SYSRETQ
#else
ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
SWAPGS /* user gsbase */
SYSRETQ
#endif
/*NOTREACHED*/
SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
_syscall_pre:
call pre_syscall
movl %eax, %r12d
testl %eax, %eax
jne _syscall_post_call
/*
* Didn't abort, so reload the syscall args and invoke the handler.
*/
movzwl T_SYSNUM(%r15), %eax
jmp _syscall_invoke
_syscall_ill:
call nosys
movq %rax, %r12
movq %rdx, %r13
jmp _syscall_post_call
_syscall_post:
STI
/*
* Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
* so that we can account for the extra work it takes us to finish.
*/
MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
_syscall_post_call:
movq %r12, %rdi
movq %r13, %rsi
call post_syscall
MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
jmp _sys_rtt
SET_SIZE(sys_syscall)
SET_SIZE(brand_sys_syscall)
#endif /* __lint */
#if defined(__lint)
/*ARGSUSED*/
void
sys_syscall32()
{}
#else /* __lint */
ENTRY_NP(brand_sys_syscall32)
SWAPGS /* kernel gsbase */
XPV_TRAP_POP
BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
jmp nopop_sys_syscall32
ALTENTRY(sys_syscall32)
SWAPGS /* kernel gsbase */
XPV_TRAP_POP
nopop_sys_syscall32:
movl %esp, %r10d
movq %gs:CPU_THREAD, %r15
movq T_STACK(%r15), %rsp
movl %eax, %eax
movl $U32CS_SEL, REGOFF_CS(%rsp)
movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
movq %r10, REGOFF_RSP(%rsp)
movl $UDS_SEL, REGOFF_SS(%rsp)
_syscall32_save:
movl %edi, REGOFF_RDI(%rsp)
movl %esi, REGOFF_RSI(%rsp)
movl %ebp, REGOFF_RBP(%rsp)
movl %ebx, REGOFF_RBX(%rsp)
movl %edx, REGOFF_RDX(%rsp)
movl %ecx, REGOFF_RCX(%rsp)
movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
movq $0, REGOFF_SAVFP(%rsp)
movq $0, REGOFF_SAVPC(%rsp)
/*
* Copy these registers here in case we end up stopped with
* someone (like, say, /proc) messing with our register state.
* We don't -restore- them unless we have to in update_sregs.
*
* Since userland -can't- change fsbase or gsbase directly,
* we don't bother to capture them here.
*/
xorl %ebx, %ebx
movw %ds, %bx
movq %rbx, REGOFF_DS(%rsp)
movw %es, %bx
movq %rbx, REGOFF_ES(%rsp)
movw %fs, %bx
movq %rbx, REGOFF_FS(%rsp)
movw %gs, %bx
movq %rbx, REGOFF_GS(%rsp)
/*
* Application state saved in the regs structure on the stack
* %eax is the syscall number
* %rsp is the thread's stack, %r15 is curthread
* REG_RSP(%rsp) is the user's stack
*/
SYSCALL_TRAPTRACE32($TT_SYSC)
movq %rsp, %rbp
movq T_LWP(%r15), %r14
ASSERT_NO_RUPDATE_PENDING(%r14)
ENABLE_INTR_FLAGS
MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
ASSERT_LWPTOREGS(%r14, %rsp)
incq %gs:CPU_STATS_SYS_SYSCALL
/*
* Make some space for MAXSYSARGS (currently 8) 32-bit args placed
* into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
* more succinctly:
*
* SA(MAXSYSARGS * sizeof (long)) == 64
*/
#define SYS_DROP 64 /* drop for args */
subq $SYS_DROP, %rsp
movb $LWP_SYS, LWP_STATE(%r14)
movq %r15, %rdi
movq %rsp, %rsi
call syscall_entry
/*
* Fetch the arguments copied onto the kernel stack and put
* them in the right registers to invoke a C-style syscall handler.
* %rax contains the handler address.
*
* Ideas for making all this go faster of course include simply
* forcibly fetching 6 arguments from the user stack under lofault
* protection, reverting to copyin_args only when watchpoints
* are in effect.
*
* (If we do this, make sure that exec and libthread leave
* enough space at the top of the stack to ensure that we'll
* never do a fetch from an invalid page.)
*
* Lots of ideas here, but they won't really help with bringup B-)
* Correctness can't wait, performance can wait a little longer ..
*/
movq %rax, %rbx
movl 0(%rsp), %edi
movl 8(%rsp), %esi
movl 0x10(%rsp), %edx
movl 0x18(%rsp), %ecx
movl 0x20(%rsp), %r8d
movl 0x28(%rsp), %r9d
call *SY_CALLC(%rbx)
movq %rbp, %rsp /* pop the args */
/*
* amd64 syscall handlers -always- return a 64-bit value in %rax.
* On the 32-bit kernel, they always return that value in %eax:%edx
* as required by the 32-bit ABI.
*
* Simulate the same behaviour by unconditionally splitting the
* return value in the same way.
*/
movq %rax, %r13
shrq $32, %r13 /* upper 32-bits into %edx */
movl %eax, %r12d /* lower 32-bits into %eax */
/*
* Optimistically assume that there's no post-syscall
* work to do. (This is to avoid having to call syscall_mstate()
* with interrupts disabled)
*/
MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
/*
* We must protect ourselves from being descheduled here;
* If we were, and we ended up on another cpu, or another
* lwp got in ahead of us, it could change the segment
* registers without us noticing before we return to userland.
*/
CLI(%r14)
CHECK_POSTSYS_NE(%r15, %r14, %ebx)
jne _full_syscall_postsys32
SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
/*
* To get back to userland, we need to put the return %rip in %rcx and
* the return %rfl in %r11d. The sysret instruction also arranges
* to fix up %cs and %ss; everything else is our responsibility.
*/
movl %r12d, %eax /* %eax: rval1 */
movl REGOFF_RBX(%rsp), %ebx
/* %ecx used for return pointer */
movl %r13d, %edx /* %edx: rval2 */
movl REGOFF_RBP(%rsp), %ebp
movl REGOFF_RSI(%rsp), %esi
movl REGOFF_RDI(%rsp), %edi
movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
movl REGOFF_RSP(%rsp), %esp
ASSERT_UPCALL_MASK_IS_SET
ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
SWAPGS /* user gsbase */
SYSRETL
SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
/*NOTREACHED*/
_full_syscall_postsys32:
STI
/*
* Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
* so that we can account for the extra work it takes us to finish.
*/
MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
movq %r15, %rdi
movq %r12, %rsi /* rval1 - %eax */
movq %r13, %rdx /* rval2 - %edx */
call syscall_exit
MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
jmp _sys_rtt
SET_SIZE(sys_syscall32)
SET_SIZE(brand_sys_syscall32)
#endif /* __lint */
/*
* System call handler via the sysenter instruction
* Used only for 32-bit system calls on the 64-bit kernel.
*
* The caller in userland has arranged that:
*
* - %eax contains the syscall number
* - %ecx contains the user %esp
* - %edx contains the return %eip
* - the user stack contains the args to the syscall
*
* Hardware and (privileged) initialization code have arranged that by
* the time the sysenter instructions completes:
*
* - %rip is pointing to sys_sysenter (below).
* - %cs and %ss are set to kernel text and stack (data) selectors.
* - %rsp is pointing at the lwp's stack
* - interrupts have been disabled.
*
* Note that we are unable to return both "rvals" to userland with
* this call, as %edx is used by the sysexit instruction.
*
* One final complication in this routine is its interaction with
* single-stepping in a debugger. For most of the system call mechanisms,
* the CPU automatically clears the single-step flag before we enter the
* kernel. The sysenter mechanism does not clear the flag, so a user
* single-stepping through a libc routine may suddenly find him/herself
* single-stepping through the kernel. To detect this, kmdb compares the
* trap %pc to the [brand_]sys_enter addresses on each single-step trap.
* If it finds that we have single-stepped to a sysenter entry point, it
* explicitly clears the flag and executes the sys_sysenter routine.
*
* One final complication in this final complication is the fact that we
* have two different entry points for sysenter: brand_sys_sysenter and
* sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping
* through the kernel with kmdb, we will eventually hit the instruction at
* sys_sysenter. kmdb cannot distinguish between that valid single-step
* and the undesirable one mentioned above. To avoid this situation, we
* simply add a jump over the instruction at sys_sysenter to make it
* impossible to single-step to it.
*/
#if defined(__lint)
void
sys_sysenter()
{}
#else /* __lint */
ENTRY_NP(brand_sys_sysenter)
SWAPGS /* kernel gsbase */
ALTENTRY(_brand_sys_sysenter_post_swapgs)
BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
/*
* Jump over sys_sysenter to allow single-stepping as described
* above.
*/
jmp _sys_sysenter_post_swapgs
ALTENTRY(sys_sysenter)
SWAPGS /* kernel gsbase */
ALTENTRY(_sys_sysenter_post_swapgs)
movq %gs:CPU_THREAD, %r15
movl $U32CS_SEL, REGOFF_CS(%rsp)
movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
pushfq
popq %r10
movl $UDS_SEL, REGOFF_SS(%rsp)
/*
* Set the interrupt flag before storing the flags to the
* flags image on the stack so we can return to user with
* interrupts enabled if we return via sys_rtt_syscall32
*/
orq $PS_IE, %r10
movq %r10, REGOFF_RFL(%rsp)
movl %edi, REGOFF_RDI(%rsp)
movl %esi, REGOFF_RSI(%rsp)
movl %ebp, REGOFF_RBP(%rsp)
movl %ebx, REGOFF_RBX(%rsp)
movl %edx, REGOFF_RDX(%rsp)
movl %ecx, REGOFF_RCX(%rsp)
movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */