forked from grate-driver/linux
-
Notifications
You must be signed in to change notification settings - Fork 0
/
ivt.S
1689 lines (1555 loc) · 51.8 KB
/
ivt.S
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* arch/ia64/kernel/ivt.S
*
* Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
* Stephane Eranian <eranian@hpl.hp.com>
* David Mosberger <davidm@hpl.hp.com>
* Copyright (C) 2000, 2002-2003 Intel Co
* Asit Mallick <asit.k.mallick@intel.com>
* Suresh Siddha <suresh.b.siddha@intel.com>
* Kenneth Chen <kenneth.w.chen@intel.com>
* Fenghua Yu <fenghua.yu@intel.com>
*
* 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
* 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
*
* Copyright (C) 2005 Hewlett-Packard Co
* Dan Magenheimer <dan.magenheimer@hp.com>
* Xen paravirtualization
* Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
* VA Linux Systems Japan K.K.
* pv_ops.
* Yaozu (Eddie) Dong <eddie.dong@intel.com>
*/
/*
* This file defines the interruption vector table used by the CPU.
* It does not include one entry per possible cause of interruption.
*
* The first 20 entries of the table contain 64 bundles each while the
* remaining 48 entries contain only 16 bundles each.
*
* The 64 bundles are used to allow inlining the whole handler for critical
* interruptions like TLB misses.
*
* For each entry, the comment is as follows:
*
* // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
* entry offset ----/ / / / /
* entry number ---------/ / / /
* size of the entry -------------/ / /
* vector name -------------------------------------/ /
* interruptions triggering this vector ----------------------/
*
* The table is 32KB in size and must be aligned on 32KB boundary.
* (The CPU ignores the 15 lower bits of the address)
*
* Table is based upon EAS2.6 (Oct 1999)
*/
#include <asm/asmmacro.h>
#include <asm/break.h>
#include <asm/kregs.h>
#include <asm/asm-offsets.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/system.h>
#include <asm/thread_info.h>
#include <asm/unistd.h>
#include <asm/errno.h>
#if 1
# define PSR_DEFAULT_BITS psr.ac
#else
# define PSR_DEFAULT_BITS 0
#endif
#if 0
/*
* This lets you track the last eight faults that occurred on the CPU. Make sure ar.k2 isn't
* needed for something else before enabling this...
*/
# define DBG_FAULT(i) mov r16=ar.k2;; shl r16=r16,8;; add r16=(i),r16;;mov ar.k2=r16
#else
# define DBG_FAULT(i)
#endif
#include "minstate.h"
#define FAULT(n) \
mov r31=pr; \
mov r19=n;; /* prepare to save predicates */ \
br.sptk.many dispatch_to_fault_handler
.section .text..ivt,"ax"
.align 32768 // align on 32KB boundary
.global ia64_ivt
ia64_ivt:
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
ENTRY(vhpt_miss)
DBG_FAULT(0)
/*
* The VHPT vector is invoked when the TLB entry for the virtual page table
* is missing. This happens only as a result of a previous
* (the "original") TLB miss, which may either be caused by an instruction
* fetch or a data access (or non-access).
*
* What we do here is normal TLB miss handing for the _original_ miss,
* followed by inserting the TLB entry for the virtual page table page
* that the VHPT walker was attempting to access. The latter gets
* inserted as long as page table entry above pte level have valid
* mappings for the faulting address. The TLB entry for the original
* miss gets inserted only if the pte entry indicates that the page is
* present.
*
* do_page_fault gets invoked in the following cases:
* - the faulting virtual address uses unimplemented address bits
* - the faulting virtual address has no valid page table mapping
*/
MOV_FROM_IFA(r16) // get address that caused the TLB miss
#ifdef CONFIG_HUGETLB_PAGE
movl r18=PAGE_SHIFT
MOV_FROM_ITIR(r25)
#endif
;;
RSM_PSR_DT // use physical addressing for data
mov r31=pr // save the predicate registers
mov r19=IA64_KR(PT_BASE) // get page table base address
shl r21=r16,3 // shift bit 60 into sign bit
shr.u r17=r16,61 // get the region number into r17
;;
shr.u r22=r21,3
#ifdef CONFIG_HUGETLB_PAGE
extr.u r26=r25,2,6
;;
cmp.ne p8,p0=r18,r26
sub r27=r26,r18
;;
(p8) dep r25=r18,r25,2,6
(p8) shr r22=r22,r27
#endif
;;
cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5?
shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit
;;
(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
srlz.d
LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
.pred.rel "mutex", p6, p7
(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
;;
(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
#ifdef CONFIG_PGTABLE_4
shr.u r28=r22,PUD_SHIFT // shift pud index into position
#else
shr.u r18=r22,PMD_SHIFT // shift pmd index into position
#endif
;;
ld8 r17=[r17] // get *pgd (may be 0)
;;
(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
#ifdef CONFIG_PGTABLE_4
dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
;;
shr.u r18=r22,PMD_SHIFT // shift pmd index into position
(p7) ld8 r29=[r28] // get *pud (may be 0)
;;
(p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL?
dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
#else
dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
#endif
;;
(p7) ld8 r20=[r17] // get *pmd (may be 0)
shr.u r19=r22,PAGE_SHIFT // shift pte index into position
;;
(p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL?
dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
;;
(p7) ld8 r18=[r21] // read *pte
MOV_FROM_ISR(r19) // cr.isr bit 32 tells us if this is an insn miss
;;
(p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared?
MOV_FROM_IHA(r22) // get the VHPT address that caused the TLB miss
;; // avoid RAW on p7
(p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss?
dep r23=0,r20,0,PAGE_SHIFT // clear low bits to get page address
;;
ITC_I_AND_D(p10, p11, r18, r24) // insert the instruction TLB entry and
// insert the data TLB entry
(p6) br.cond.spnt.many page_fault // handle bad address/page not present (page fault)
MOV_TO_IFA(r22, r24)
#ifdef CONFIG_HUGETLB_PAGE
MOV_TO_ITIR(p8, r25, r24) // change to default page-size for VHPT
#endif
/*
* Now compute and insert the TLB entry for the virtual page table. We never
* execute in a page table page so there is no need to set the exception deferral
* bit.
*/
adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23
;;
ITC_D(p7, r24, r25)
;;
#ifdef CONFIG_SMP
/*
* Tell the assemblers dependency-violation checker that the above "itc" instructions
* cannot possibly affect the following loads:
*/
dv_serialize_data
/*
* Re-check pagetable entry. If they changed, we may have received a ptc.g
* between reading the pagetable and the "itc". If so, flush the entry we
* inserted and retry. At this point, we have:
*
* r28 = equivalent of pud_offset(pgd, ifa)
* r17 = equivalent of pmd_offset(pud, ifa)
* r21 = equivalent of pte_offset(pmd, ifa)
*
* r29 = *pud
* r20 = *pmd
* r18 = *pte
*/
ld8 r25=[r21] // read *pte again
ld8 r26=[r17] // read *pmd again
#ifdef CONFIG_PGTABLE_4
ld8 r19=[r28] // read *pud again
#endif
cmp.ne p6,p7=r0,r0
;;
cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change
#ifdef CONFIG_PGTABLE_4
cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change
#endif
mov r27=PAGE_SHIFT<<2
;;
(p6) ptc.l r22,r27 // purge PTE page translation
(p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change
;;
(p6) ptc.l r16,r27 // purge translation
#endif
mov pr=r31,-1 // restore predicate registers
RFI
END(vhpt_miss)
.org ia64_ivt+0x400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0400 Entry 1 (size 64 bundles) ITLB (21)
ENTRY(itlb_miss)
DBG_FAULT(1)
/*
* The ITLB handler accesses the PTE via the virtually mapped linear
* page table. If a nested TLB miss occurs, we switch into physical
* mode, walk the page table, and then re-execute the PTE read and
* go on normally after that.
*/
MOV_FROM_IFA(r16) // get virtual address
mov r29=b0 // save b0
mov r31=pr // save predicates
.itlb_fault:
MOV_FROM_IHA(r17) // get virtual address of PTE
movl r30=1f // load nested fault continuation point
;;
1: ld8 r18=[r17] // read *pte
;;
mov b0=r29
tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
(p6) br.cond.spnt page_fault
;;
ITC_I(p0, r18, r19)
;;
#ifdef CONFIG_SMP
/*
* Tell the assemblers dependency-violation checker that the above "itc" instructions
* cannot possibly affect the following loads:
*/
dv_serialize_data
ld8 r19=[r17] // read *pte again and see if same
mov r20=PAGE_SHIFT<<2 // setup page size for purge
;;
cmp.ne p7,p0=r18,r19
;;
(p7) ptc.l r16,r20
#endif
mov pr=r31,-1
RFI
END(itlb_miss)
.org ia64_ivt+0x0800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
ENTRY(dtlb_miss)
DBG_FAULT(2)
/*
* The DTLB handler accesses the PTE via the virtually mapped linear
* page table. If a nested TLB miss occurs, we switch into physical
* mode, walk the page table, and then re-execute the PTE read and
* go on normally after that.
*/
MOV_FROM_IFA(r16) // get virtual address
mov r29=b0 // save b0
mov r31=pr // save predicates
dtlb_fault:
MOV_FROM_IHA(r17) // get virtual address of PTE
movl r30=1f // load nested fault continuation point
;;
1: ld8 r18=[r17] // read *pte
;;
mov b0=r29
tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
(p6) br.cond.spnt page_fault
;;
ITC_D(p0, r18, r19)
;;
#ifdef CONFIG_SMP
/*
* Tell the assemblers dependency-violation checker that the above "itc" instructions
* cannot possibly affect the following loads:
*/
dv_serialize_data
ld8 r19=[r17] // read *pte again and see if same
mov r20=PAGE_SHIFT<<2 // setup page size for purge
;;
cmp.ne p7,p0=r18,r19
;;
(p7) ptc.l r16,r20
#endif
mov pr=r31,-1
RFI
END(dtlb_miss)
.org ia64_ivt+0x0c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
ENTRY(alt_itlb_miss)
DBG_FAULT(3)
MOV_FROM_IFA(r16) // get address that caused the TLB miss
movl r17=PAGE_KERNEL
MOV_FROM_IPSR(p0, r21)
movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
mov r31=pr
;;
#ifdef CONFIG_DISABLE_VHPT
shr.u r22=r16,61 // get the region number into r21
;;
cmp.gt p8,p0=6,r22 // user mode
;;
THASH(p8, r17, r16, r23)
;;
MOV_TO_IHA(p8, r17, r23)
(p8) mov r29=b0 // save b0
(p8) br.cond.dptk .itlb_fault
#endif
extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
shr.u r18=r16,57 // move address bit 61 to bit 4
;;
andcm r18=0x10,r18 // bit 4=~address-bit(61)
cmp.ne p8,p0=r0,r23 // psr.cpl != 0?
or r19=r17,r19 // insert PTE control bits into r19
;;
or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
(p8) br.cond.spnt page_fault
;;
ITC_I(p0, r19, r18) // insert the TLB entry
mov pr=r31,-1
RFI
END(alt_itlb_miss)
.org ia64_ivt+0x1000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
ENTRY(alt_dtlb_miss)
DBG_FAULT(4)
MOV_FROM_IFA(r16) // get address that caused the TLB miss
movl r17=PAGE_KERNEL
MOV_FROM_ISR(r20)
movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
MOV_FROM_IPSR(p0, r21)
mov r31=pr
mov r24=PERCPU_ADDR
;;
#ifdef CONFIG_DISABLE_VHPT
shr.u r22=r16,61 // get the region number into r21
;;
cmp.gt p8,p0=6,r22 // access to region 0-5
;;
THASH(p8, r17, r16, r25)
;;
MOV_TO_IHA(p8, r17, r25)
(p8) mov r29=b0 // save b0
(p8) br.cond.dptk dtlb_fault
#endif
cmp.ge p10,p11=r16,r24 // access to per_cpu_data?
tbit.z p12,p0=r16,61 // access to region 6?
mov r25=PERCPU_PAGE_SHIFT << 2
mov r26=PERCPU_PAGE_SIZE
nop.m 0
nop.b 0
;;
(p10) mov r19=IA64_KR(PER_CPU_DATA)
(p11) and r19=r19,r16 // clear non-ppn fields
extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field
tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on?
tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
;;
(p10) sub r19=r19,r26
MOV_TO_ITIR(p10, r25, r24)
cmp.ne p8,p0=r0,r23
(p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field
(p12) dep r17=-1,r17,4,1 // set ma=UC for region 6 addr
(p8) br.cond.spnt page_fault
dep r21=-1,r21,IA64_PSR_ED_BIT,1
;;
or r19=r19,r17 // insert PTE control bits into r19
MOV_TO_IPSR(p6, r21, r24)
;;
ITC_D(p7, r19, r18) // insert the TLB entry
mov pr=r31,-1
RFI
END(alt_dtlb_miss)
.org ia64_ivt+0x1400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
ENTRY(nested_dtlb_miss)
/*
* In the absence of kernel bugs, we get here when the virtually mapped linear
* page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
* Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page
* table is missing, a nested TLB miss fault is triggered and control is
* transferred to this point. When this happens, we lookup the pte for the
* faulting address by walking the page table in physical mode and return to the
* continuation point passed in register r30 (or call page_fault if the address is
* not mapped).
*
* Input: r16: faulting address
* r29: saved b0
* r30: continuation address
* r31: saved pr
*
* Output: r17: physical address of PTE of faulting address
* r29: saved b0
* r30: continuation address
* r31: saved pr
*
* Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared)
*/
RSM_PSR_DT // switch to using physical data addressing
mov r19=IA64_KR(PT_BASE) // get the page table base address
shl r21=r16,3 // shift bit 60 into sign bit
MOV_FROM_ITIR(r18)
;;
shr.u r17=r16,61 // get the region number into r17
extr.u r18=r18,2,6 // get the faulting page size
;;
cmp.eq p6,p7=5,r17 // is faulting address in region 5?
add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
;;
shr.u r22=r16,r22
shr.u r18=r16,r18
(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
srlz.d
LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
.pred.rel "mutex", p6, p7
(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
;;
(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
#ifdef CONFIG_PGTABLE_4
shr.u r18=r22,PUD_SHIFT // shift pud index into position
#else
shr.u r18=r22,PMD_SHIFT // shift pmd index into position
#endif
;;
ld8 r17=[r17] // get *pgd (may be 0)
;;
(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
;;
#ifdef CONFIG_PGTABLE_4
(p7) ld8 r17=[r17] // get *pud (may be 0)
shr.u r18=r22,PMD_SHIFT // shift pmd index into position
;;
(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
;;
#endif
(p7) ld8 r17=[r17] // get *pmd (may be 0)
shr.u r19=r22,PAGE_SHIFT // shift pte index into position
;;
(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
(p6) br.cond.spnt page_fault
mov b0=r30
br.sptk.many b0 // return to continuation point
END(nested_dtlb_miss)
.org ia64_ivt+0x1800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
ENTRY(ikey_miss)
DBG_FAULT(6)
FAULT(6)
END(ikey_miss)
.org ia64_ivt+0x1c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
ENTRY(dkey_miss)
DBG_FAULT(7)
FAULT(7)
END(dkey_miss)
.org ia64_ivt+0x2000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
ENTRY(dirty_bit)
DBG_FAULT(8)
/*
* What we do here is to simply turn on the dirty bit in the PTE. We need to
* update both the page-table and the TLB entry. To efficiently access the PTE,
* we address it through the virtual page table. Most likely, the TLB entry for
* the relevant virtual page table page is still present in the TLB so we can
* normally do this without additional TLB misses. In case the necessary virtual
* page table TLB entry isn't present, we take a nested TLB miss hit where we look
* up the physical address of the L3 PTE and then continue at label 1 below.
*/
MOV_FROM_IFA(r16) // get the address that caused the fault
movl r30=1f // load continuation point in case of nested fault
;;
THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
mov r29=b0 // save b0 in case of nested fault
mov r31=pr // save pr
#ifdef CONFIG_SMP
mov r28=ar.ccv // save ar.ccv
;;
1: ld8 r18=[r17]
;; // avoid RAW on r18
mov ar.ccv=r18 // set compare value for cmpxchg
or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
;;
(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
mov r24=PAGE_SHIFT<<2
;;
(p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
;;
ITC_D(p6, r25, r18) // install updated PTE
;;
/*
* Tell the assemblers dependency-violation checker that the above "itc" instructions
* cannot possibly affect the following loads:
*/
dv_serialize_data
ld8 r18=[r17] // read PTE again
;;
cmp.eq p6,p7=r18,r25 // is it same as the newly installed
;;
(p7) ptc.l r16,r24
mov b0=r29 // restore b0
mov ar.ccv=r28
#else
;;
1: ld8 r18=[r17]
;; // avoid RAW on r18
or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
mov b0=r29 // restore b0
;;
st8 [r17]=r18 // store back updated PTE
ITC_D(p0, r18, r16) // install updated PTE
#endif
mov pr=r31,-1 // restore pr
RFI
END(dirty_bit)
.org ia64_ivt+0x2400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
ENTRY(iaccess_bit)
DBG_FAULT(9)
// Like Entry 8, except for instruction access
MOV_FROM_IFA(r16) // get the address that caused the fault
movl r30=1f // load continuation point in case of nested fault
mov r31=pr // save predicates
#ifdef CONFIG_ITANIUM
/*
* Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
*/
MOV_FROM_IPSR(p0, r17)
;;
MOV_FROM_IIP(r18)
tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set?
;;
(p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa
#endif /* CONFIG_ITANIUM */
;;
THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
mov r29=b0 // save b0 in case of nested fault)
#ifdef CONFIG_SMP
mov r28=ar.ccv // save ar.ccv
;;
1: ld8 r18=[r17]
;;
mov ar.ccv=r18 // set compare value for cmpxchg
or r25=_PAGE_A,r18 // set the accessed bit
tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
;;
(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
mov r24=PAGE_SHIFT<<2
;;
(p6) cmp.eq p6,p7=r26,r18 // Only if page present
;;
ITC_I(p6, r25, r26) // install updated PTE
;;
/*
* Tell the assemblers dependency-violation checker that the above "itc" instructions
* cannot possibly affect the following loads:
*/
dv_serialize_data
ld8 r18=[r17] // read PTE again
;;
cmp.eq p6,p7=r18,r25 // is it same as the newly installed
;;
(p7) ptc.l r16,r24
mov b0=r29 // restore b0
mov ar.ccv=r28
#else /* !CONFIG_SMP */
;;
1: ld8 r18=[r17]
;;
or r18=_PAGE_A,r18 // set the accessed bit
mov b0=r29 // restore b0
;;
st8 [r17]=r18 // store back updated PTE
ITC_I(p0, r18, r16) // install updated PTE
#endif /* !CONFIG_SMP */
mov pr=r31,-1
RFI
END(iaccess_bit)
.org ia64_ivt+0x2800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
ENTRY(daccess_bit)
DBG_FAULT(10)
// Like Entry 8, except for data access
MOV_FROM_IFA(r16) // get the address that caused the fault
movl r30=1f // load continuation point in case of nested fault
;;
THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
mov r31=pr
mov r29=b0 // save b0 in case of nested fault)
#ifdef CONFIG_SMP
mov r28=ar.ccv // save ar.ccv
;;
1: ld8 r18=[r17]
;; // avoid RAW on r18
mov ar.ccv=r18 // set compare value for cmpxchg
or r25=_PAGE_A,r18 // set the dirty bit
tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
;;
(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present
mov r24=PAGE_SHIFT<<2
;;
(p6) cmp.eq p6,p7=r26,r18 // Only if page is present
;;
ITC_D(p6, r25, r26) // install updated PTE
/*
* Tell the assemblers dependency-violation checker that the above "itc" instructions
* cannot possibly affect the following loads:
*/
dv_serialize_data
;;
ld8 r18=[r17] // read PTE again
;;
cmp.eq p6,p7=r18,r25 // is it same as the newly installed
;;
(p7) ptc.l r16,r24
mov ar.ccv=r28
#else
;;
1: ld8 r18=[r17]
;; // avoid RAW on r18
or r18=_PAGE_A,r18 // set the accessed bit
;;
st8 [r17]=r18 // store back updated PTE
ITC_D(p0, r18, r16) // install updated PTE
#endif
mov b0=r29 // restore b0
mov pr=r31,-1
RFI
END(daccess_bit)
.org ia64_ivt+0x2c00
/////////////////////////////////////////////////////////////////////////////////////////
// 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
ENTRY(break_fault)
/*
* The streamlined system call entry/exit paths only save/restore the initial part
* of pt_regs. This implies that the callers of system-calls must adhere to the
* normal procedure calling conventions.
*
* Registers to be saved & restored:
* CR registers: cr.ipsr, cr.iip, cr.ifs
* AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr
* others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15
* Registers to be restored only:
* r8-r11: output value from the system call.
*
* During system call exit, scratch registers (including r15) are modified/cleared
* to prevent leaking bits from kernel to user level.
*/
DBG_FAULT(11)
mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
MOV_FROM_IPSR(p0, r29) // M2 (12 cyc)
mov r31=pr // I0 (2 cyc)
MOV_FROM_IIM(r17) // M2 (2 cyc)
mov.m r27=ar.rsc // M2 (12 cyc)
mov r18=__IA64_BREAK_SYSCALL // A
mov.m ar.rsc=0 // M2
mov.m r21=ar.fpsr // M2 (12 cyc)
mov r19=b6 // I0 (2 cyc)
;;
mov.m r23=ar.bspstore // M2 (12 cyc)
mov.m r24=ar.rnat // M2 (5 cyc)
mov.i r26=ar.pfs // I0 (2 cyc)
invala // M0|1
nop.m 0 // M
mov r20=r1 // A save r1
nop.m 0
movl r30=sys_call_table // X
MOV_FROM_IIP(r28) // M2 (2 cyc)
cmp.eq p0,p7=r18,r17 // I0 is this a system call?
(p7) br.cond.spnt non_syscall // B no ->
//
// From this point on, we are definitely on the syscall-path
// and we can use (non-banked) scratch registers.
//
///////////////////////////////////////////////////////////////////////
mov r1=r16 // A move task-pointer to "addl"-addressable reg
mov r2=r16 // A setup r2 for ia64_syscall_setup
add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags
adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
adds r15=-1024,r15 // A subtract 1024 from syscall number
mov r3=NR_syscalls - 1
;;
ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr
shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS
cmp.leu p6,p7=r15,r3 // A syscall number in range?
;;
lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS
(p6) ld8 r30=[r30] // M0|1 load address of syscall entry point
tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT?
mov.m ar.bspstore=r22 // M2 switch to kernel RBS
cmp.eq p8,p9=2,r8 // A isr.ei==2?
;;
(p8) mov r8=0 // A clear ei to 0
(p7) movl r30=sys_ni_syscall // X
(p8) adds r28=16,r28 // A switch cr.iip to next bundle
(p9) adds r8=1,r8 // A increment ei to next slot
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
;;
mov b6=r30 // I0 setup syscall handler branch reg early
#else
nop.i 0
;;
#endif
mov.m r25=ar.unat // M2 (5 cyc)
dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr
adds r15=1024,r15 // A restore original syscall number
//
// If any of the above loads miss in L1D, we'll stall here until
// the data arrives.
//
///////////////////////////////////////////////////////////////////////
st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
MOV_FROM_ITC(p0, p14, r30, r18) // M get cycle for accounting
#else
mov b6=r30 // I0 setup syscall handler branch reg early
#endif
cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already?
and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit
mov r18=ar.bsp // M2 (12 cyc)
(pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
;;
.back_from_break_fixup:
(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
br.call.sptk.many b7=ia64_syscall_setup // B
1:
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
// mov.m r30=ar.itc is called in advance, and r13 is current
add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A
add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A
(pKStk) br.cond.spnt .skip_accounting // B unlikely skip
;;
ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // M get last stamp
ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // M time at leave
;;
ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // M cumulated stime
ld8 r21=[r17] // M cumulated utime
sub r22=r19,r18 // A stime before leave
;;
st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // M update stamp
sub r18=r30,r19 // A elapsed time in user
;;
add r20=r20,r22 // A sum stime
add r21=r21,r18 // A sum utime
;;
st8 [r16]=r20 // M update stime
st8 [r17]=r21 // M update utime
;;
.skip_accounting:
#endif
mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
nop 0
BSW_1(r2, r14) // B (6 cyc) regs are saved, switch to bank 1
;;
SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r16) // M2 now it's safe to re-enable intr.-collection
// M0 ensure interruption collection is on
movl r3=ia64_ret_from_syscall // X
;;
mov rp=r3 // I0 set the real return addr
(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
SSM_PSR_I(p15, p15, r16) // M2 restore psr.i
(p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
// NOT REACHED
///////////////////////////////////////////////////////////////////////
// On entry, we optimistically assumed that we're coming from user-space.
// For the rare cases where a system-call is done from within the kernel,
// we fix things up at this point:
.break_fixup:
add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
mov ar.rnat=r24 // M2 restore kernel's AR.RNAT
;;
mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE
br.cond.sptk .back_from_break_fixup
END(break_fault)
.org ia64_ivt+0x3000
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
ENTRY(interrupt)
/* interrupt handler has become too big to fit this area. */
br.sptk.many __interrupt
END(interrupt)
.org ia64_ivt+0x3400
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3400 Entry 13 (size 64 bundles) Reserved
DBG_FAULT(13)
FAULT(13)
.org ia64_ivt+0x3800
/////////////////////////////////////////////////////////////////////////////////////////
// 0x3800 Entry 14 (size 64 bundles) Reserved
DBG_FAULT(14)
FAULT(14)
/*
* There is no particular reason for this code to be here, other than that
* there happens to be space here that would go unused otherwise. If this
* fault ever gets "unreserved", simply moved the following code to a more
* suitable spot...
*
* ia64_syscall_setup() is a separate subroutine so that it can
* allocate stacked registers so it can safely demine any
* potential NaT values from the input registers.
*
* On entry:
* - executing on bank 0 or bank 1 register set (doesn't matter)
* - r1: stack pointer
* - r2: current task pointer
* - r3: preserved
* - r11: original contents (saved ar.pfs to be saved)
* - r12: original contents (sp to be saved)
* - r13: original contents (tp to be saved)
* - r15: original contents (syscall # to be saved)
* - r18: saved bsp (after switching to kernel stack)
* - r19: saved b6
* - r20: saved r1 (gp)
* - r21: saved ar.fpsr
* - r22: kernel's register backing store base (krbs_base)
* - r23: saved ar.bspstore
* - r24: saved ar.rnat
* - r25: saved ar.unat
* - r26: saved ar.pfs
* - r27: saved ar.rsc
* - r28: saved cr.iip
* - r29: saved cr.ipsr
* - r30: ar.itc for accounting (don't touch)
* - r31: saved pr
* - b0: original contents (to be saved)
* On exit:
* - p10: TRUE if syscall is invoked with more than 8 out
* registers or r15's Nat is true
* - r1: kernel's gp
* - r3: preserved (same as on entry)
* - r8: -EINVAL if p10 is true
* - r12: points to kernel stack
* - r13: points to current task
* - r14: preserved (same as on entry)
* - p13: preserved
* - p15: TRUE if interrupts need to be re-enabled
* - ar.fpsr: set to kernel settings
* - b6: preserved (same as on entry)
*/
#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
GLOBAL_ENTRY(ia64_syscall_setup)
#if PT(B6) != 0
# error This code assumes that b6 is the first field in pt_regs.
#endif
st8 [r1]=r19 // save b6
add r16=PT(CR_IPSR),r1 // initialize first base pointer
add r17=PT(R11),r1 // initialize second base pointer
;;
alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable
st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr
tnat.nz p8,p0=in0
st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11
tnat.nz p9,p0=in1
(pKStk) mov r18=r0 // make sure r18 isn't NaT
;;
st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs
st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip
mov r28=b0 // save b0 (2 cyc)
;;
st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat
dep r19=0,r19,38,26 // clear all bits but 0..37 [I0]
(p8) mov in0=-1
;;
st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs
extr.u r11=r19,7,7 // I0 // get sol of ar.pfs
and r8=0x7f,r19 // A // get sof of ar.pfs
st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0
(p9) mov in1=-1
;;
(pUStk) sub r18=r18,r22 // r18=RSE.ndirty*8
tnat.nz p10,p0=in2
add r11=8,r11
;;
(pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field
(pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field
tnat.nz p11,p0=in3
;;
(p10) mov in2=-1
tnat.nz p12,p0=in4 // [I0]
(p11) mov in3=-1
;;
(pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat
(pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore
shl r18=r18,16 // compute ar.rsc to be used for "loadrs"
;;
st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates
st8 [r17]=r28,PT(R1)-PT(B0) // save b0
tnat.nz p13,p0=in5 // [I0]
;;
st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs"
st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1
(p12) mov in4=-1