xref: /illumos-gate/usr/src/uts/sun4v/ml/mach_locore.S (revision 55fea89dcaa64928bed4327112404dcb3e07b79f)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/asm_linkage.h>
27#include <sys/intreg.h>
28#include <sys/ivintr.h>
29#include <sys/mmu.h>
30#include <sys/machpcb.h>
31#include <sys/machtrap.h>
32#include <sys/machlock.h>
33#include <sys/fdreg.h>
34#include <sys/vis.h>
35#include <sys/traptrace.h>
36#include <sys/panic.h>
37#include <sys/machasi.h>
38#include <sys/privregs.h>
39#include <sys/hypervisor_api.h>
40#include <sys/clock.h>
41
42#include "assym.h"
43
44
45!
46! REGOFF must add up to allow double word access to r_tstate.
47! PCB_WBUF must also be aligned.
48!
49#if (REGOFF & 7) != 0
50#error "struct regs not aligned"
51#endif
52
53/*
54 * Absolute external symbols.
55 * On the sun4u we put the panic buffer in the third and fourth pages.
56 * We set things up so that the first 2 pages of KERNELBASE is illegal
57 * to act as a redzone during copyin/copyout type operations. One of
58 * the reasons the panic buffer is allocated in low memory to
59 * prevent being overwritten during booting operations (besides
60 * the fact that it is small enough to share pages with others).
61 */
62
63	.seg	".data"
64	.global	panicbuf
65
66PROM	= 0xFFE00000			! address of prom virtual area
67panicbuf = SYSBASE32 + PAGESIZE		! address of panic buffer
68
69	.type	panicbuf, #object
70	.size	panicbuf, PANICBUFSIZE
71
72/*
73 * Absolute external symbol - intr_vec_table.
74 *
75 * With new bus structures supporting a larger number of interrupt
76 * numbers, the interrupt vector table, intr_vec_table[] has been
77 * moved out of kernel nucleus and allocated after panicbuf.
78 */
79	.global intr_vec_table
80
81intr_vec_table = SYSBASE32 + PAGESIZE + PANICBUFSIZE ! address of interrupt table
82
83	.type	intr_vec_table, #object
84	.size	intr_vec_table, MAXIVNUM * CPTRSIZE + MAX_RSVD_IV * IV_SIZE + MAX_RSVD_IVX * (IV_SIZE + CPTRSIZE * (NCPU - 1))
85
86/*
87 * The thread 0 stack. This must be the first thing in the data
88 * segment (other than an sccs string) so that we don't stomp
89 * on anything important if the stack overflows. We get a
90 * red zone below this stack for free when the kernel text is
91 * write protected.
92 */
93
94	.global	t0stack
95	.align	16
96	.type	t0stack, #object
97t0stack:
98	.skip	T0STKSZ			! thread 0 stack
99t0stacktop:
100	.size	t0stack, T0STKSZ
101
102/*
103 * cpu0 and its ptl1_panic stack.  The cpu structure must be allocated
104 * on a single page for ptl1_panic's physical address accesses.
105 */
106	.global	cpu0
107	.align	MMU_PAGESIZE
108cpu0:
109	.type	cpu0, #object
110	.skip	CPU_ALLOC_SIZE
111	.size	cpu0, CPU_ALLOC_SIZE
112
113	.global t0
114	.align	PTR24_ALIGN		! alignment for mutex.
115	.type	t0, #object
116t0:
117	.skip	THREAD_SIZE		! thread 0
118	.size	t0, THREAD_SIZE
119
120	.global	trap_trace_ctl
121	.global	htrap_tr0
122	.global htrap_trace_bufsize
123
124	.align	64
125trap_trace_ctl:
126	.skip	NCPU * TRAPTR_SIZE	! NCPU control headers
127htrap_tr0:
128	.skip	HTRAP_TSIZE		! one buffer for the boot cpu
129	.align	4
130htrap_trace_bufsize:
131	.word	HTRAP_TSIZE		! default hv trap buffer size
132
133#ifdef	TRAPTRACE
134	.global	trap_tr0
135	.global trap_trace_bufsize
136	.global	trap_freeze
137	.global	trap_freeze_pc
138
139	.align	4
140trap_trace_bufsize:
141	.word	TRAP_TSIZE		! default trap buffer size
142trap_freeze:
143	.word	0
144
145	.align	16
146trap_tr0:
147	.skip	TRAP_TSIZE		! one buffer for the boot cpu
148
149/*
150 * When an assertion in TRACE_PTR was failed, %pc is saved in trap_freeze_pc to
151 * show in which TRACE_PTR the assertion failure happened.
152 */
153	.align	8
154trap_freeze_pc:
155	.nword	0
156#endif	/* TRAPTRACE */
157
158	.align 4
159	.seg	".text"
160
161#ifdef	NOPROM
162	.global availmem
163availmem:
164	.word	0
165#endif	/* NOPROM */
166
167	.align	8
168_local_p1275cis:
169	.nword	0
170
171	.seg	".data"
172
173	.global nwindows, nwin_minus_one, winmask
174nwindows:
175	.word   8
176nwin_minus_one:
177	.word   7
178winmask:
179	.word	8
180
181	.global	afsrbuf
182afsrbuf:
183	.word	0,0,0,0
184
185/*
186 * System initialization
187 *
188 * Our contract with the boot prom specifies that the MMU is on and the
189 * first 16 meg of memory is mapped with a level-1 pte.  We are called
190 * with p1275cis ptr in %o0 and kdi_dvec in %o1; we start execution
191 * directly from physical memory, so we need to get up into our proper
192 * addresses quickly: all code before we do this must be position
193 * independent.
194 *
195 * NB: Above is not true for boot/stick kernel, the only thing mapped is
196 * the text+data+bss. The kernel is loaded directly into KERNELBASE.
197 *
198 * 	entry, the romvec pointer (romp) is the first argument;
199 * 	  i.e., %o0.
200 * 	the bootops vector is in the third argument (%o1)
201 *
202 * Our tasks are:
203 * 	save parameters
204 * 	construct mappings for KERNELBASE (not needed for boot/stick kernel)
205 * 	hop up into high memory           (not needed for boot/stick kernel)
206 * 	initialize stack pointer
207 * 	initialize trap base register
208 * 	initialize window invalid mask
209 * 	initialize psr (with traps enabled)
210 * 	figure out all the module type stuff
211 * 	tear down the 1-1 mappings
212 * 	dive into main()
213 */
214	ENTRY_NP(_start)
215	!
216	! Stash away our arguments in memory.
217	!
218	sethi	%hi(_local_p1275cis), %g1
219	stn	%o4, [%g1 + %lo(_local_p1275cis)]
220
221	!
222	! Initialize CPU state registers
223	!
224	wrpr	%g0, PSTATE_KERN, %pstate
225	wr	%g0, %g0, %fprs
226
227	!
228	! call krtld to link the world together
229	!
230	call	kobj_start
231	mov	%o4, %o0
232
233	! Write 0x1f (MAX_REG_WINDOWS) to %cwp and read back to get
234	! the actual implemented nwin - 1 value
235	rdpr	%cwp, %g2		! save current %cwp
236	wrpr	%g0, 0x1f, %cwp
237	rdpr	%cwp, %g1		! %g1 = nwin - 1
238	wrpr	%g0, %g2, %cwp		! restore current %cwp
239
240	!
241	! Stuff some memory cells related to numbers of windows.
242	!
243	sethi	%hi(nwin_minus_one), %g2
244	st	%g1, [%g2 + %lo(nwin_minus_one)]
245	inc	%g1
246	sethi	%hi(nwindows), %g2
247	st	%g1, [%g2 + %lo(nwindows)]
248	dec	%g1
249	mov	-2, %g2
250	sll	%g2, %g1, %g2
251	sethi	%hi(winmask), %g4
252	st	%g2, [%g4 + %lo(winmask)]
253
254	!
255	! save a pointer to obp's tba for later use by kmdb
256	!
257	rdpr	%tba, %g1
258	set	boot_tba, %g2
259	stx	%g1, [%g2]
260
261	!
262	! copy obp's breakpoint trap entry to obp_bpt
263	!
264	rdpr	%tba, %g1
265	set	T_SOFTWARE_TRAP | ST_MON_BREAKPOINT, %g2
266	sll	%g2, 5, %g2
267	or	%g1, %g2, %g1
268	set	obp_bpt, %g2
269	ldx	[%g1], %g3
270	stx	%g3, [%g2]
271	flush	%g2
272	ldx	[%g1 + 8], %g3
273	stx	%g3, [%g2 + 8]
274	flush	%g2 + 8
275	ldx	[%g1 + 16], %g3
276	stx	%g3, [%g2 + 16]
277	flush	%g2 + 16
278	ldx	[%g1 + 24], %g3
279	stx	%g3, [%g2 + 24]
280	flush	%g2 + 24
281
282	!
283	! Initialize thread 0's stack.
284	!
285	set	t0stacktop, %g1		! setup kernel stack pointer
286	sub	%g1, SA(KFPUSIZE+GSR_SIZE), %g2
287	and	%g2, 0x3f, %g3
288	sub	%g2, %g3, %o1
289	sub	%o1, SA(MPCBSIZE) + STACK_BIAS, %sp
290
291	!
292	! Initialize global thread register.
293	!
294	set	t0, THREAD_REG
295
296	!
297	! Fill in enough of the cpu structure so that
298	! the wbuf management code works. Make sure the
299	! boot cpu is inserted in cpu[] based on cpuid.
300	!
301	CPU_INDEX(%g2, %g1)
302	sll	%g2, CPTRSHIFT, %g2		! convert cpuid to cpu[] offset
303	set	cpu0, %o0			! &cpu0
304	set	cpu, %g1			! &cpu[]
305	stn	%o0, [%g1 + %g2]		! cpu[cpuid] = &cpu0
306
307	stn	%o0, [THREAD_REG + T_CPU]	! threadp()->t_cpu = cpu[cpuid]
308	stn	THREAD_REG, [%o0 + CPU_THREAD]	! cpu[cpuid]->cpu_thread = threadp()
309
310
311	!  We do NOT need to bzero our BSS...boot has already done it for us.
312	!  Just need to reference edata so that we don't break /dev/ksyms
313	set	edata, %g0
314
315	!
316	! Call mlsetup with address of prototype user registers.
317	!
318	call	mlsetup
319	add	%sp, REGOFF + STACK_BIAS, %o0
320
321#if (REGOFF != MPCB_REGS)
322#error "hole in struct machpcb between frame and regs?"
323#endif
324
325	!
326	! Now call main.  We will return as process 1 (init).
327	!
328	call	main
329	nop
330
331	!
332	! Main should never return.
333	!
334	set	.mainretmsg, %o0
335	call	panic
336	nop
337	SET_SIZE(_start)
338
339.mainretmsg:
340	.asciz	"main returned"
341	.align	4
342
343
344/*
345 * Generic system trap handler.
346 *
347 * Some kernel trap handlers save themselves from buying a window by
348 * borrowing some of sys_trap's unused locals. %l0 thru %l3 may be used
349 * for this purpose, as user_rtt and priv_rtt do not depend on them.
350 * %l4 thru %l7 should NOT be used this way.
351 *
352 * Entry Conditions:
353 * 	%pstate		am:0 priv:1 ie:0
354 * 	%gl		global level  1
355 *
356 * Register Inputs:
357 * 	%g1		pc of trap handler
358 * 	%g2, %g3	args for handler
359 * 	%g4		desired %pil (-1 means current %pil)
360 * 	%g5, %g6	destroyed
361 * 	%g7		saved
362 *
363 * Register Usage:
364 * 	%l0, %l1	temps
365 * 	%l3		saved %g1
366 * 	%l6		curthread for user traps, %pil for priv traps
367 * 	%l7		regs
368 *
369 * Called function prototype variants:
370 *
371 *	func(struct regs *rp);
372 * 	func(struct regs *rp, uintptr_t arg1 [%g2], uintptr_t arg2 [%g3])
373 *	func(struct regs *rp, uintptr_t arg1 [%g2],
374 *	    uint32_t arg2 [%g3.l], uint32_t arg3 [%g3.h])
375 *	func(struct regs *rp, uint32_t arg1 [%g2.l],
376 *	    uint32_t arg2 [%g3.l], uint32_t arg3 [%g3.h], uint32_t [%g2.h])
377 */
378
379	ENTRY_NP(sys_trap)
380#ifdef DEBUG
381	! Assert gl == 1
382	rdpr	%gl, %g5
383	cmp	%g5, 1
384	bne,a,pn %xcc, ptl1_panic
385	  mov	PTL1_BAD_GL, %g1
386#endif
387
388	!
389	! force tl=1, update %cwp, branch to correct handler
390	!
391
392	wrpr	%g0, 1, %tl
393	rdpr	%tstate, %g5
394	btst	TSTATE_PRIV, %g5
395	and	%g5, TSTATE_CWP, %g6
396	bnz,pn	%xcc, priv_trap
397	wrpr	%g0, %g6, %cwp
398
399	ALTENTRY(user_trap)
400	!
401	! user trap
402	!
403	! make all windows clean for kernel
404	! buy a window using the current thread's stack
405	!
406#ifdef DEBUG
407	! Assert gl == 1
408	rdpr	%gl, %g5
409	cmp	%g5, 1
410	bne,a,pn %xcc, ptl1_panic
411	  mov	PTL1_BAD_GL, %g1
412#endif
413	sethi	%hi(nwin_minus_one), %g5
414	ld	[%g5 + %lo(nwin_minus_one)], %g5
415	wrpr	%g0, %g5, %cleanwin
416	CPU_ADDR(%g5, %g6)
417	ldn	[%g5 + CPU_THREAD], %g5
418	ldn	[%g5 + T_STACK], %g6
419	sub	%g6, STACK_BIAS, %g6
420	save	%g6, 0, %sp
421	!
422	! set window registers so that current windows are "other" windows
423	!
424	rdpr	%canrestore, %l0
425	rdpr	%wstate, %l1
426	wrpr	%g0, 0, %canrestore
427	sllx	%l1, WSTATE_SHIFT, %l1
428	wrpr	%l1, WSTATE_K64, %wstate
429	wrpr	%g0, %l0, %otherwin
430	!
431	! set pcontext to run kernel
432	!
433	mov	KCONTEXT, %l0
434	mov	MMU_PCONTEXT, %l1
435	stxa	%l0, [%l1]ASI_MMU_CTX
436	! Ensure new ctx takes effect by the time the "done" (below) completes
437	membar	#Sync
438
439	set	utl0, %g6		! bounce to utl0
440have_win:
441#ifdef DEBUG
442	CPU_ADDR(%o1, %o2)
443	add	%o1, CPU_MCPU, %o1
444	ld	[%o1 + MCPU_KWBUF_FULL], %o2
445	tst	%o2
446	bnz,a,pn %icc, ptl1_panic
447	  mov	PTL1_BAD_WTRAP, %g1
448#endif /* DEBUG */
449	SYSTRAP_TRACE(%o1, %o2, %o3)
450
451
452	!
453	! at this point we have a new window we can play in,
454	! and %g6 is the label we want done to bounce to
455	!
456	! save needed current globals
457	!
458	mov	%g1, %l3	! pc
459	mov	%g2, %o1	! arg #1
460	mov	%g3, %o2	! arg #2
461	srlx	%g3, 32, %o3	! pseudo arg #3
462	srlx	%g2, 32, %o4	! pseudo arg #4
463	mov	%g5, %l6	! curthread if user trap, %pil if priv trap
464	!
465	! save trap state on stack
466	!
467	add	%sp, REGOFF + STACK_BIAS, %l7
468	rdpr	%tpc, %l0
469	rdpr	%tnpc, %l1
470	rdpr	%tstate, %l2
471	stn	%l0, [%l7 + PC_OFF]
472	stn	%l1, [%l7 + nPC_OFF]
473	stx	%l2, [%l7 + TSTATE_OFF]
474	!
475	! setup pil
476	!
477	brlz,pt		%g4, 1f
478	nop
479#ifdef DEBUG
480	!
481	! ASSERT(%g4 >= %pil).
482	!
483	rdpr	%pil, %l0
484	cmp	%g4, %l0
485	bge,pt	%xcc, 0f
486	nop				! yes, nop; to avoid anull
487	set	bad_g4_called, %l3
488	mov	1, %o1
489	st	%o1, [%l3]
490	set	bad_g4, %l3		! pc
491	set	sys_trap_wrong_pil, %o1	! arg #1
492	mov	%g4, %o2		! arg #2
493	ba	1f			! stay at the current %pil
494	mov	%l0, %o3		! arg #3
4950:
496#endif /* DEBUG */
497	wrpr		%g0, %g4, %pil
4981:
499	!
500	! set trap regs to execute in kernel at %g6
501	! done resumes execution there
502	!
503	wrpr	%g0, %g6, %tnpc
504	rdpr	%cwp, %l0
505	set	TSTATE_KERN, %l1
506	wrpr	%l1, %l0, %tstate
507	done
508	/* NOTREACHED */
509	SET_SIZE(user_trap)
510	SET_SIZE(sys_trap)
511
512#define	KWBUF64_TO_STACK(SBP,SPP,TMP) \
513	ldx	[SBP + (0*8)], TMP; \
514	stx	TMP, [SPP + V9BIAS64 + 0]; \
515	ldx	[SBP + (1*8)], TMP; \
516	stx	TMP, [SPP + V9BIAS64 + 8]; \
517	ldx	[SBP + (2*8)], TMP; \
518	stx	TMP, [SPP + V9BIAS64 + 16]; \
519	ldx	[SBP + (3*8)], TMP; \
520	stx	TMP, [SPP + V9BIAS64 + 24]; \
521	ldx	[SBP + (4*8)], TMP; \
522	stx	TMP, [SPP + V9BIAS64 + 32]; \
523	ldx	[SBP + (5*8)], TMP; \
524	stx	TMP, [SPP + V9BIAS64 + 40]; \
525	ldx	[SBP + (6*8)], TMP; \
526	stx	TMP, [SPP + V9BIAS64 + 48]; \
527	ldx	[SBP + (7*8)], TMP; \
528	stx	TMP, [SPP + V9BIAS64 + 56]; \
529	ldx	[SBP + (8*8)], TMP; \
530	stx	TMP, [SPP + V9BIAS64 + 64]; \
531	ldx	[SBP + (9*8)], TMP; \
532	stx	TMP, [SPP + V9BIAS64 + 72]; \
533	ldx	[SBP + (10*8)], TMP; \
534	stx	TMP, [SPP + V9BIAS64 + 80]; \
535	ldx	[SBP + (11*8)], TMP; \
536	stx	TMP, [SPP + V9BIAS64 + 88]; \
537	ldx	[SBP + (12*8)], TMP; \
538	stx	TMP, [SPP + V9BIAS64 + 96]; \
539	ldx	[SBP + (13*8)], TMP; \
540	stx	TMP, [SPP + V9BIAS64 + 104]; \
541	ldx	[SBP + (14*8)], TMP; \
542	stx	TMP, [SPP + V9BIAS64 + 112]; \
543	ldx	[SBP + (15*8)], TMP; \
544	stx	TMP, [SPP + V9BIAS64 + 120];
545
546#define	KWBUF32_TO_STACK(SBP,SPP,TMP) \
547	lduw	[SBP + (0 * 4)], TMP; \
548	stw	TMP, [SPP + 0]; \
549	lduw	[SBP + (1 * 4)], TMP; \
550	stw	TMP, [SPP + (1 * 4)]; \
551	lduw	[SBP + (2 * 4)], TMP; \
552	stw	TMP, [SPP + (2 * 4)]; \
553	lduw	[SBP + (3 * 4)], TMP; \
554	stw	TMP, [SPP + (3 * 4)]; \
555	lduw	[SBP + (4 * 4)], TMP; \
556	stw	TMP, [SPP + (4 * 4)]; \
557	lduw	[SBP + (5 * 4)], TMP; \
558	stw	TMP, [SPP + (5 * 4)]; \
559	lduw	[SBP + (6 * 4)], TMP; \
560	stw	TMP, [SPP + (6 * 4)]; \
561	lduw	[SBP + (7 * 4)], TMP; \
562	stw	TMP, [SPP + (7 * 4)]; \
563	lduw	[SBP + (8 * 4)], TMP; \
564	stw	TMP, [SPP + (8 * 4)]; \
565	lduw	[SBP + (9 * 4)], TMP; \
566	stw	TMP, [SPP + (9 * 4)]; \
567	lduw	[SBP + (10 * 4)], TMP; \
568	stw	TMP, [SPP + (10 * 4)]; \
569	lduw	[SBP + (11 * 4)], TMP; \
570	stw	TMP, [SPP + (11 * 4)]; \
571	lduw	[SBP + (12 * 4)], TMP; \
572	stw	TMP, [SPP + (12 * 4)]; \
573	lduw	[SBP + (13 * 4)], TMP; \
574	stw	TMP, [SPP + (13 * 4)]; \
575	lduw	[SBP + (14 * 4)], TMP; \
576	stw	TMP, [SPP + (14 * 4)]; \
577	lduw	[SBP + (15 * 4)], TMP; \
578	stw	TMP, [SPP + (15 * 4)];
579
580#define	COPY_KWBUF_TO_STACK(TMP1,TMP2,TMP3)		\
581	CPU_ADDR(TMP2, TMP3)				;\
582	add	TMP2, CPU_MCPU, TMP2			;\
583	ld	[TMP2 + MCPU_KWBUF_FULL], TMP3		;\
584	brz,pt	TMP3, 2f				;\
585	nop						;\
586	st	%g0, [TMP2 + MCPU_KWBUF_FULL]		;\
587	set	MCPU_KWBUF_SP, TMP3			;\
588	ldn	[TMP2 + TMP3], TMP3			;\
589	set	MCPU_KWBUF, TMP1			;\
590	btst	1, TMP3					;\
591	bz,pn	%xcc, 3f				;\
592	add	TMP2, TMP1, TMP2			;\
593	KWBUF64_TO_STACK(TMP2, TMP3, TMP1)		;\
594	ba,a	2f					;\
5953:							;\
596	KWBUF32_TO_STACK(TMP2, TMP3, TMP1)		;\
5972:
598
599	ENTRY_NP(prom_trap)
600	!
601	! prom trap switches the stack to 32-bit
602	! if we took a trap from a 64-bit window
603	! Then buys a window on the current stack.
604	!
605	save	%sp, -SA64(REGOFF + REGSIZE), %sp
606					/* 32 bit frame, 64 bit sized */
607	COPY_KWBUF_TO_STACK(%o1, %o2, %o3)
608	set	ptl0, %g6
609	ba,a,pt	%xcc, have_win
610	SET_SIZE(prom_trap)
611
612	ENTRY_NP(priv_trap)
613	!
614	! kernel trap
615	! buy a window on the current stack
616	!
617	! is the trap PC in the range allocated to Open Firmware?
618	rdpr	%tpc, %g5
619	set	OFW_END_ADDR, %g6
620	cmp	%g5, %g6
621	bgu,a,pn %xcc, 1f
622	  rdpr	%pil, %g5
623	set	OFW_START_ADDR, %g6
624	cmp	%g5, %g6
625	bgeu,pn	%xcc, prom_trap
626	  rdpr	%pil, %g5
6271:
628	set	ktl0, %g6
629	save	%sp, -SA(REGOFF + REGSIZE), %sp
630	COPY_KWBUF_TO_STACK(%o1, %o2, %o3)
631	ba,a,pt	%xcc, have_win
632	SET_SIZE(priv_trap)
633
634/*
635 * FILL_32bit_rtt/FILL_64bit_rtt fills a 32/64-bit-wide register window
636 * from a 32/64-bit * wide address space via the designated asi.
637 * It is used to fill windows in user_rtt to avoid going above TL 2.
638 */
639/* TODO: Use the faster FILL based on FILL_32bit_asi/FILL_64bit_asi */
640#define	FILL_32bit_rtt(asi_num)				\
641	mov	asi_num, %asi					;\
642	rdpr	%cwp, %g1					;\
643	dec	%g1						;\
644	wrpr	%g1, %cwp					;\
645	srl	%sp, 0, %sp					;\
646	lda	[%sp + 0]%asi, %l0				;\
647	lda	[%sp + 4]%asi, %l1				;\
648	lda	[%sp + 8]%asi, %l2				;\
649	lda	[%sp + 12]%asi, %l3				;\
650	lda	[%sp + 16]%asi, %l4				;\
651	lda	[%sp + 20]%asi, %l5				;\
652	lda	[%sp + 24]%asi, %l6				;\
653	lda	[%sp + 28]%asi, %l7				;\
654	lda	[%sp + 32]%asi, %i0				;\
655	lda	[%sp + 36]%asi, %i1				;\
656	lda	[%sp + 40]%asi, %i2				;\
657	lda	[%sp + 44]%asi, %i3				;\
658	lda	[%sp + 48]%asi, %i4				;\
659	lda	[%sp + 52]%asi, %i5				;\
660	lda	[%sp + 56]%asi, %i6				;\
661	lda	[%sp + 60]%asi, %i7				;\
662	restored						;\
663	add	%g1, 1, %g1					;\
664	wrpr	%g1, %cwp
665
666#define	FILL_64bit_rtt(asi_num)				\
667	mov	asi_num, %asi					;\
668	rdpr	%cwp, %g1					;\
669	sub	%g1, 1, %g1					;\
670	wrpr	%g1, %cwp					;\
671	ldxa	[%sp + V9BIAS64 + 0]%asi, %l0			;\
672	ldxa	[%sp + V9BIAS64 + 8]%asi, %l1			;\
673	ldxa	[%sp + V9BIAS64 + 16]%asi, %l2			;\
674	ldxa	[%sp + V9BIAS64 + 24]%asi, %l3			;\
675	ldxa	[%sp + V9BIAS64 + 32]%asi, %l4			;\
676	ldxa	[%sp + V9BIAS64 + 40]%asi, %l5			;\
677	ldxa	[%sp + V9BIAS64 + 48]%asi, %l6			;\
678	ldxa	[%sp + V9BIAS64 + 56]%asi, %l7			;\
679	ldxa	[%sp + V9BIAS64 + 64]%asi, %i0			;\
680	ldxa	[%sp + V9BIAS64 + 72]%asi, %i1			;\
681	ldxa	[%sp + V9BIAS64 + 80]%asi, %i2			;\
682	ldxa	[%sp + V9BIAS64 + 88]%asi, %i3			;\
683	ldxa	[%sp + V9BIAS64 + 96]%asi, %i4			;\
684	ldxa	[%sp + V9BIAS64 + 104]%asi, %i5			;\
685	ldxa	[%sp + V9BIAS64 + 112]%asi, %i6			;\
686	ldxa	[%sp + V9BIAS64 + 120]%asi, %i7			;\
687	restored						;\
688	add	%g1, 1, %g1					;\
689	wrpr	%g1, %cwp
690
691	ENTRY_NP(utl0)
692	SAVE_GLOBALS(%l7)
693	SAVE_OUTS(%l7)
694	mov	%l6, THREAD_REG
695	wrpr	%g0, PSTATE_KERN, %pstate	! enable ints
696	jmpl	%l3, %o7			! call trap handler
697	mov	%l7, %o0
698	!
699	ALTENTRY(user_rtt)
700	!
701	! Register inputs
702	!	%l7 - regs
703	!
704	! disable interrupts and check for ASTs and wbuf restores
705	! keep cpu_base_spl in %l4
706	!
707	wrpr	%g0, PIL_MAX, %pil
708	ldn	[THREAD_REG + T_CPU], %l0
709	ld	[%l0 + CPU_BASE_SPL], %l4
710
711	ldub	[THREAD_REG + T_ASTFLAG], %l2
712	brz,pt	%l2, 1f
713	ld	[%sp + STACK_BIAS + MPCB_WBCNT], %l3
714	!
715	! call trap to do ast processing
716	!
717	wrpr	%g0, %l4, %pil			! pil = cpu_base_spl
718	mov	%l7, %o0
719	call	trap
720	  mov	T_AST, %o2
721	ba,a,pt	%xcc, user_rtt
7221:
723	brz,pt	%l3, 2f
724	mov	THREAD_REG, %l6
725	!
726	! call restore_wbuf to push wbuf windows to stack
727	!
728	wrpr	%g0, %l4, %pil			! pil = cpu_base_spl
729	mov	%l7, %o0
730	call	trap
731	  mov	T_FLUSH_PCB, %o2
732	ba,a,pt	%xcc, user_rtt
7332:
734#ifdef TRAPTRACE
735	TRACE_RTT(TT_SYS_RTT_USER, %l0, %l1, %l2, %l3)
736#endif /* TRAPTRACE */
737	ld	[%sp + STACK_BIAS + MPCB_WSTATE], %l3	! get wstate
738
739	!
740	! restore user globals and outs
741	!
742	rdpr	%pstate, %l1
743	wrpr	%l1, PSTATE_IE, %pstate
744	RESTORE_GLOBALS(%l7)
745	! switch to global set 1, saving THREAD_REG in %l6
746	wrpr	%g0, 1, %gl
747	mov	%sp, %g6	! remember the mpcb pointer in %g6
748	RESTORE_OUTS(%l7)
749	!
750	! set %pil from cpu_base_spl
751	!
752	wrpr	%g0, %l4, %pil
753	!
754	! raise tl (now using nucleus context)
755	! set pcontext to scontext for user execution
756	!
757	wrpr	%g0, 1, %tl
758
759	mov	MMU_SCONTEXT, %g1
760	ldxa	[%g1]ASI_MMU_CTX, %g2
761	mov	MMU_PCONTEXT, %g1
762	stxa	%g2, [%g1]ASI_MMU_CTX
763	!
764	! If shared context support is not enabled, then the next six
765	! instructions will be patched with nop instructions.
766	!
767	.global sfmmu_shctx_user_rtt_patch
768sfmmu_shctx_user_rtt_patch:
769	!
770	! On processors which support multiple contexts, writing to
771	! pcontext0 automatically updates pcontext1 for backwards
772	! compatibility. So, if scontext0 & scontext1 are the same
773	! a write to pcontext0 is sufficient.
774	!
775	mov	MMU_SCONTEXT1, %g1
776	ldxa	[%g1]ASI_MMU_CTX, %g3
777	cmp	%g2, %g3
778	beq,pt	%xcc, no_pctx1_update
779	  mov	MMU_PCONTEXT1, %g1
780	stxa	%g3, [%g1]ASI_MMU_CTX
781
782no_pctx1_update:
783	! Ensure new ctxs take effect by the time the "retry" (below) completes
784	membar	#Sync
785
786	!
787	! setup trap regs
788	!
789	ldn	[%l7 + PC_OFF], %g1
790	ldn	[%l7 + nPC_OFF], %g2
791	ldx	[%l7 + TSTATE_OFF], %l0
792	andn	%l0, TSTATE_CWP, %g7
793	wrpr	%g1, %tpc
794	wrpr	%g2, %tnpc
795	!
796	! switch "other" windows back to "normal" windows and
797	! restore to window we originally trapped in
798	!
799	rdpr	%otherwin, %g1
800	wrpr	%g0, 0, %otherwin
801	add	%l3, WSTATE_CLEAN_OFFSET, %l3	! convert to "clean" wstate
802	wrpr	%g0, %l3, %wstate
803	wrpr	%g0, %g1, %canrestore
804	!
805	! First attempt to restore from the watchpoint saved register window
806	tst	%g1
807	bne,a	1f
808	  clrn	[%g6 + STACK_BIAS + MPCB_RSP0]
809	tst	%fp
810	be,a	1f
811	  clrn	[%g6 + STACK_BIAS + MPCB_RSP0]
812	! test for user return window in pcb
813	ldn	[%g6 + STACK_BIAS + MPCB_RSP0], %g1
814	cmp	%fp, %g1
815	bne	1f
816	  clrn	[%g6 + STACK_BIAS + MPCB_RSP0]
817	restored
818	restore
819	! restore from user return window
820	RESTORE_V9WINDOW(%g6 + STACK_BIAS + MPCB_RWIN0)
821	!
822	! Attempt to restore from the scond watchpoint saved register window
823	tst	%fp
824	be,a	2f
825	  clrn	[%g6 + STACK_BIAS + MPCB_RSP1]
826	ldn	[%g6 + STACK_BIAS + MPCB_RSP1], %g1
827	cmp	%fp, %g1
828	bne	2f
829	  clrn	[%g6 + STACK_BIAS + MPCB_RSP1]
830	restored
831	restore
832	RESTORE_V9WINDOW(%g6 + STACK_BIAS + MPCB_RWIN1)
833	save
834	b,a	2f
8351:
836	rdpr	%canrestore, %g1
837	brnz	%g1, 3f
838	nop			! no trap, use restore directly
839	rdpr	%cwp, %g1
840	wrpr	%g1, %g7, %tstate	! needed by wbuf recovery code
841	! hand craft the restore to avoid getting to TL > 2
842	rdpr	%wstate, %g1
843	btst	1, %g1
844	beq	4f
845	nop
846	.global	rtt_fill_start
847rtt_fill_start:
848	FILL_32bit_rtt(ASI_AIUP)
849	ba,a	3f
8504:
851	FILL_64bit_rtt(ASI_AIUP)
852	.global	rtt_fill_end
853rtt_fill_end:
8543:
855	restore				! should not trap
8562:
857	!
858	! set %cleanwin to %canrestore
859	! set %tstate to the correct %cwp
860	! retry resumes user execution
861	!
862	rdpr	%canrestore, %g1
863	wrpr	%g0, %g1, %cleanwin
864	rdpr	%cwp, %g1
865	wrpr	%g1, %g7, %tstate
866	retry
867	/* NOTREACHED */
868	SET_SIZE(user_rtt)
869	SET_SIZE(utl0)
870
871	ENTRY_NP(ptl0)
872	SAVE_GLOBALS(%l7)
873	SAVE_OUTS(%l7)
874	CPU_ADDR(%g5, %g6)
875	ldn	[%g5 + CPU_THREAD], THREAD_REG
876	wrpr	%g0, PSTATE_KERN, %pstate	! enable ints
877	jmpl	%l3, %o7			! call trap handler
878	mov	%l7, %o0
879	!
880	ALTENTRY(prom_rtt)
881#ifdef TRAPTRACE
882	TRACE_RTT(TT_SYS_RTT_PROM, %l0, %l1, %l2, %l3)
883#endif /* TRAPTRACE */
884	ba,pt	%xcc, common_rtt
885	mov	THREAD_REG, %l0
886	SET_SIZE(prom_rtt)
887	SET_SIZE(ptl0)
888
889	ENTRY_NP(ktl0)
890	/*
891	 * THREAD_REG cannot be restored in fault_32bit_fn1 since
892	 * sun4v cannot safely lower %gl then raise it again.
893	 */
894	CPU_ADDR(%l0, %l1)
895	ldn	[%l0 + CPU_THREAD], THREAD_REG
896	SAVE_GLOBALS(%l7)
897	SAVE_OUTS(%l7)				! for the call bug workaround
898	wrpr	%g0, PSTATE_KERN, %pstate	! enable ints
899	jmpl	%l3, %o7			! call trap handler
900	mov	%l7, %o0
901	!
902	ALTENTRY(priv_rtt)
903#ifdef TRAPTRACE
904	TRACE_RTT(TT_SYS_RTT_PRIV, %l0, %l1, %l2, %l3)
905#endif /* TRAPTRACE */
906	!
907	! Register inputs
908	!	%l7 - regs
909	!	%l6 - trap %pil
910	!
911	! Check for a kernel preemption request
912	!
913	ldn	[THREAD_REG + T_CPU], %l0
914	ldub	[%l0 + CPU_KPRUNRUN], %l0
915	brz,pt	%l0, 1f
916	nop
917
918	!
919	! Attempt to preempt
920	!
921	ldstub	[THREAD_REG + T_PREEMPT_LK], %l0	! load preempt lock
922	brnz,pn	%l0, 1f			! can't call kpreempt if this thread is
923	nop				!   already in it...
924
925	call	kpreempt
926	mov	%l6, %o0		! pass original interrupt level
927
928	stub	%g0, [THREAD_REG + T_PREEMPT_LK]	! nuke the lock
929
930	rdpr	%pil, %o0		! compare old pil level
931	cmp	%l6, %o0		!   with current pil level
932	movg	%xcc, %o0, %l6		! if current is lower, drop old pil
9331:
934	!
935	! If we interrupted the mutex_owner_running() critical region we
936	! must reset ! the PC and nPC back to the beginning to prevent missed
937	! wakeups.  ! See the comments in mutex_exit() for details.
938	!
939	ldn	[%l7 + PC_OFF], %l0
940	set	mutex_owner_running_critical_start, %l1
941	sub	%l0, %l1, %l0
942	cmp	%l0, mutex_owner_running_critical_size
943	bgeu,pt	%xcc, 2f
944	mov	THREAD_REG, %l0
945	stn	%l1, [%l7 + PC_OFF]	! restart mutex_owner_running()
946	add	%l1, 4, %l1
947	ba,pt	%xcc, common_rtt
948	stn	%l1, [%l7 + nPC_OFF]
949
9502:
951	!
952	! If we interrupted the mutex_exit() critical region we must reset
953	! the PC and nPC back to the beginning to prevent missed wakeups.
954	! See the comments in mutex_exit() for details.
955	!
956	ldn	[%l7 + PC_OFF], %l0
957	set	mutex_exit_critical_start, %l1
958	sub	%l0, %l1, %l0
959	cmp	%l0, mutex_exit_critical_size
960	bgeu,pt	%xcc, common_rtt
961	mov	THREAD_REG, %l0
962	stn	%l1, [%l7 + PC_OFF]	! restart mutex_exit()
963	add	%l1, 4, %l1
964	stn	%l1, [%l7 + nPC_OFF]
965
966common_rtt:
967	!
968	! restore globals and outs
969	!
970	rdpr	%pstate, %l1
971	wrpr	%l1, PSTATE_IE, %pstate
972	RESTORE_GLOBALS(%l7)
973	! switch to global set 1
974	wrpr	%g0, 1, %gl
975	RESTORE_OUTS(%l7)
976	!
977	! set %pil from max(old pil, cpu_base_spl)
978	!
979	ldn	[%l0 + T_CPU], %l0
980	ld	[%l0 + CPU_BASE_SPL], %l0
981	cmp	%l6, %l0
982	movg	%xcc, %l6, %l0
983	wrpr	%g0, %l0, %pil
984	!
985	! raise tl
986	! setup trap regs
987	! restore to window we originally trapped in
988	!
989	wrpr	%g0, 1, %tl
990	ldn	[%l7 + PC_OFF], %g1
991	ldn	[%l7 + nPC_OFF], %g2
992	ldx	[%l7 + TSTATE_OFF], %l0
993	andn	%l0, TSTATE_CWP, %g7
994	wrpr	%g1, %tpc
995	wrpr	%g2, %tnpc
996	rdpr	%canrestore, %g1
997	brnz	%g1, 3f
998	nop			! no trap, use restore directly
999	rdpr	%cwp, %g1
1000	wrpr	%g1, %g7, %tstate	! needed by wbuf recovery code
1001	! hand craft the restore to avoid getting to TL > 2
1002	FILL_64bit_rtt(ASI_N)
10033:
1004	restore
1005	!
1006	! set %tstate to the correct %cwp
1007	! retry resumes prom execution
1008	!
1009	rdpr	%cwp, %g1
1010	wrpr	%g1, %g7, %tstate
1011	retry
1012	/* NOTREACHED */
1013	SET_SIZE(priv_rtt)
1014	SET_SIZE(ktl0)
1015
1016#ifdef DEBUG
1017	.seg	".data"
1018	.align	4
1019
1020	.global bad_g4_called
1021bad_g4_called:
1022	.word	0
1023
1024sys_trap_wrong_pil:
1025	.asciz	"sys_trap: %g4(%d) is lower than %pil(%d)"
1026	.align	4
1027	.seg	".text"
1028
1029	ENTRY_NP(bad_g4)
1030	mov	%o1, %o0
1031	mov	%o2, %o1
1032	call	panic
1033	mov	%o3, %o2
1034	SET_SIZE(bad_g4)
1035#endif /* DEBUG */
1036
1037/*
1038 * sys_tl1_panic can be called by traps at tl1 which
1039 * really want to panic, but need the rearrangement of
1040 * the args as provided by this wrapper routine.
1041 */
1042	ENTRY_NP(sys_tl1_panic)
1043	mov	%o1, %o0
1044	mov	%o2, %o1
1045	call	panic
1046	mov	%o3, %o2
1047	SET_SIZE(sys_tl1_panic)
1048
1049
1050/*
1051 * Flush all windows to memory, except for the one we entered in.
1052 * We do this by doing NWINDOW-2 saves then the same number of restores.
1053 * This leaves the WIM immediately before window entered in.
1054 * This is used for context switching.
1055 */
1056
1057	ENTRY_NP(flush_windows)
1058	retl
1059	flushw
1060	SET_SIZE(flush_windows)
1061
1062	ENTRY_NP(debug_flush_windows)
1063	set	nwindows, %g1
1064	ld	[%g1], %g1
1065	mov	%g1, %g2
1066
10671:
1068	save	%sp, -WINDOWSIZE, %sp
1069	brnz	%g2, 1b
1070	dec	%g2
1071
1072	mov	%g1, %g2
10732:
1074	restore
1075	brnz	%g2, 2b
1076	dec	%g2
1077
1078	retl
1079	nop
1080
1081	SET_SIZE(debug_flush_windows)
1082
1083/*
1084 * flush user windows to memory.
1085 */
1086
1087	ENTRY_NP(flush_user_windows)
1088	rdpr	%otherwin, %g1
1089	brz	%g1, 3f
1090	clr	%g2
10911:
1092	save	%sp, -WINDOWSIZE, %sp
1093	rdpr	%otherwin, %g1
1094	brnz	%g1, 1b
1095	add	%g2, 1, %g2
10962:
1097	sub	%g2, 1, %g2		! restore back to orig window
1098	brnz	%g2, 2b
1099	restore
11003:
1101	retl
1102	nop
1103	SET_SIZE(flush_user_windows)
1104
1105/*
1106 * Throw out any user windows in the register file.
1107 * Used by setregs (exec) to clean out old user.
1108 * Used by sigcleanup to remove extraneous windows when returning from a
1109 * signal.
1110 */
1111
1112	ENTRY_NP(trash_user_windows)
1113	rdpr	%otherwin, %g1
1114	brz	%g1, 3f			! no user windows?
1115	ldn	[THREAD_REG + T_STACK], %g5
1116
1117	!
1118	! There are old user windows in the register file. We disable ints
1119	! and increment cansave so that we don't overflow on these windows.
1120	! Also, this sets up a nice underflow when first returning to the
1121	! new user.
1122	!
1123	rdpr	%pstate, %g2
1124	wrpr	%g2, PSTATE_IE, %pstate
1125	rdpr	%cansave, %g3
1126	rdpr	%otherwin, %g1		! re-read in case of interrupt
1127	add	%g3, %g1, %g3
1128	wrpr	%g0, 0, %otherwin
1129	wrpr	%g0, %g3, %cansave
1130	wrpr	%g0, %g2, %pstate
11313:
1132	retl
1133 	clr     [%g5 + MPCB_WBCNT]       ! zero window buffer cnt
1134	SET_SIZE(trash_user_windows)
1135
1136
1137/*
1138 * Setup g7 via the CPU data structure.
1139 */
1140
1141	ENTRY_NP(set_tbr)
1142	retl
1143	ta	72		! no tbr, stop simulation
1144	SET_SIZE(set_tbr)
1145
1146
1147#define	PTL1_SAVE_WINDOW(RP)						\
1148	stxa	%l0, [RP + RW64_LOCAL + (0 * RW64_LOCAL_INCR)] %asi;	\
1149	stxa	%l1, [RP + RW64_LOCAL + (1 * RW64_LOCAL_INCR)] %asi;	\
1150	stxa	%l2, [RP + RW64_LOCAL + (2 * RW64_LOCAL_INCR)] %asi;	\
1151	stxa	%l3, [RP + RW64_LOCAL + (3 * RW64_LOCAL_INCR)] %asi;	\
1152	stxa	%l4, [RP + RW64_LOCAL + (4 * RW64_LOCAL_INCR)] %asi;	\
1153	stxa	%l5, [RP + RW64_LOCAL + (5 * RW64_LOCAL_INCR)] %asi;	\
1154	stxa	%l6, [RP + RW64_LOCAL + (6 * RW64_LOCAL_INCR)] %asi;	\
1155	stxa	%l7, [RP + RW64_LOCAL + (7 * RW64_LOCAL_INCR)] %asi;	\
1156	stxa	%i0, [RP + RW64_IN + (0 * RW64_IN_INCR)] %asi;		\
1157	stxa	%i1, [RP + RW64_IN + (1 * RW64_IN_INCR)] %asi;		\
1158	stxa	%i2, [RP + RW64_IN + (2 * RW64_IN_INCR)] %asi;		\
1159	stxa	%i3, [RP + RW64_IN + (3 * RW64_IN_INCR)] %asi;		\
1160	stxa	%i4, [RP + RW64_IN + (4 * RW64_IN_INCR)] %asi;		\
1161	stxa	%i5, [RP + RW64_IN + (5 * RW64_IN_INCR)] %asi;		\
1162	stxa	%i6, [RP + RW64_IN + (6 * RW64_IN_INCR)] %asi;		\
1163	stxa	%i7, [RP + RW64_IN + (7 * RW64_IN_INCR)] %asi
1164#define	PTL1_NEXT_WINDOW(scr)	\
1165	add	scr, RWIN64SIZE, scr
1166
1167#define	PTL1_RESET_RWINDOWS(scr)			\
1168	sethi	%hi(nwin_minus_one), scr;		\
1169	ld	[scr + %lo(nwin_minus_one)], scr;	\
1170	wrpr	scr, %cleanwin;				\
1171	dec	scr;					\
1172	wrpr	scr, %cansave;				\
1173	wrpr	%g0, %canrestore;			\
1174	wrpr	%g0, %otherwin
1175
1176#define	PTL1_DCACHE_LINE_SIZE	4	/* small enough for all CPUs */
1177
1178/*
1179 * ptl1_panic is called when the kernel detects that it is in an invalid state
1180 * and the trap level is greater than 0.  ptl1_panic is responsible to save the
1181 * current CPU state, to restore the CPU state to normal, and to call panic.
1182 * The CPU state must be saved reliably without causing traps.  ptl1_panic saves
1183 * it in the ptl1_state structure, which is a member of the machcpu structure.
1184 * In order to access the ptl1_state structure without causing traps, physical
1185 * addresses are used so that we can avoid MMU miss traps.  The restriction of
1186 * physical memory accesses is that the ptl1_state structure must be on a single
1187 * physical page.  This is because (1) a single physical address for each
1188 * ptl1_state structure is needed and (2) it simplifies physical address
1189 * calculation for each member of the structure.
1190 * ptl1_panic is a likely spot for stack overflows to wind up; thus, the current
1191 * stack may not be usable.  In order to call panic reliably in such a state,
1192 * each CPU needs a dedicated ptl1 panic stack.
1193 * CPU_ALLOC_SIZE, which is defined to be MMU_PAGESIZE, is used to allocate the
1194 * cpu structure and a ptl1 panic stack.  They are put together on the same page
1195 * for memory space efficiency.  The low address part is used for the cpu
1196 * structure, and the high address part is for a ptl1 panic stack.
1197 * The cpu_pa array holds the physical addresses of the allocated cpu structures,
1198 * as the cpu array holds their virtual addresses.
1199 *
1200 * %g1 reason to be called
1201 * %g2 broken
1202 * %g3 broken
1203 */
1204	ENTRY_NP(ptl1_panic)
1205	!
1206	! increment the entry counter.
1207	! save CPU state if this is the first entry.
1208	!
1209	CPU_PADDR(%g2, %g3);
1210	add	%g2, CPU_PTL1, %g2		! pstate = &CPU->mcpu.ptl1_state
1211	wr	%g0, ASI_MEM, %asi		! physical address access
1212	!
1213	! pstate->ptl1_entry_count++
1214	!
1215	lduwa	[%g2 + PTL1_ENTRY_COUNT] %asi, %g3
1216	add	%g3, 1, %g3
1217	stuwa	%g3, [%g2 + PTL1_ENTRY_COUNT] %asi
1218	!
1219	! CPU state saving is skipped from the 2nd entry to ptl1_panic since we
1220	! do not want to clobber the state from the original failure.  panic()
1221	! is responsible for handling multiple or recursive panics.
1222	!
1223	cmp	%g3, 2				! if (ptl1_entry_count >= 2)
1224	bge,pn	%icc, state_saved		!	goto state_saved
1225	  add	%g2, PTL1_REGS, %g3		! %g3 = &pstate->ptl1_regs[0]
1226	!
1227	! save CPU state
1228	!
1229save_cpu_state:
1230	! save current global registers
1231	! so that all them become available for use
1232	!
1233	stxa	%o1, [%g3 + PTL1_RWINDOW] %asi		! save %o1
1234	stxa	%o2, [%g3 + PTL1_RWINDOW + 8] %asi	! save %o2
1235	stxa	%o3, [%g3 + PTL1_RWINDOW + 16] %asi	! save %o3
1236	rdpr	%gl, %o1
1237	add	%g3, PTL1_GREGS, %o2		! %o4 = &ptl1_gregs[0]
1238	mov	%g3, %o3
12396:
1240	stxa	%o1, [%o2 + PTL1_GL] %asi
1241	stxa	%g1, [%o2 + PTL1_G1] %asi
1242	stxa	%g2, [%o2 + PTL1_G2] %asi
1243	stxa	%g3, [%o2 + PTL1_G3] %asi
1244	stxa	%g4, [%o2 + PTL1_G4] %asi
1245	stxa	%g5, [%o2 + PTL1_G5] %asi
1246	stxa	%g6, [%o2 + PTL1_G6] %asi
1247	stxa	%g7, [%o2 + PTL1_G7] %asi
1248	add	%o2, PTL1_GREGS_INCR, %o2
1249	deccc	%o1
1250	brgez,a,pt %o1, 6b
1251	wrpr	%o1, %gl
1252	!
1253	! restore %g3, %o1, %o2 and %o3
1254	!
1255	mov	%o3, %g3
1256	ldxa	[%g3 + PTL1_RWINDOW] %asi, %o1
1257	ldxa	[%g3 + PTL1_RWINDOW + 8] %asi, %o2
1258	ldxa	[%g3 + PTL1_RWINDOW + 16] %asi, %o3
1259	!
1260	! %tl, %tt, %tstate, %tpc, %tnpc for each TL
1261	!
1262	rdpr	%tl, %g1
1263	brz	%g1, 1f				! if(trap_level == 0) -------+
1264	add	%g3, PTL1_TRAP_REGS, %g4	! %g4 = &ptl1_trap_regs[0];  !
12650:						! -----------<----------+    !
1266	stwa	%g1, [%g4 + PTL1_TL] %asi				!    !
1267	rdpr	%tt, %g5						!    !
1268	stwa	%g5, [%g4 + PTL1_TT] %asi				!    !
1269	rdpr	%tstate, %g5						!    !
1270	stxa	%g5, [%g4 + PTL1_TSTATE] %asi				!    !
1271	rdpr	%tpc, %g5						!    !
1272	stxa	%g5, [%g4 + PTL1_TPC] %asi				!    !
1273	rdpr	%tnpc, %g5						!    !
1274	stxa	%g5, [%g4 + PTL1_TNPC] %asi				!    !
1275	add	%g4, PTL1_TRAP_REGS_INCR, %g4				!    !
1276	deccc	%g1							!    !
1277	bnz,a,pt %icc, 0b			! if(trap_level != 0) --+    !
1278	  wrpr	%g1, %tl						     !
12791:						! ----------<----------------+
1280	!
1281	! %pstate, %pil, SOFTINT, (S)TICK
1282	! Pending interrupts is also cleared in order to avoid a recursive call
1283	! to ptl1_panic in case the interrupt handler causes a panic.
1284	!
1285	rdpr	%pil, %g1
1286	stba	%g1, [%g3 + PTL1_PIL] %asi
1287	rdpr	%pstate, %g1
1288	stha	%g1, [%g3 + PTL1_PSTATE] %asi
1289	rd	SOFTINT, %g1
1290	sta	%g1, [%g3 + PTL1_SOFTINT] %asi
1291	wr	%g1, CLEAR_SOFTINT
1292	RD_TICKSTICK_FLAG(%g1, %g4, traptrace_use_stick)
1293	stxa	%g1, [%g3 + PTL1_TICK] %asi
1294
1295	MMU_FAULT_STATUS_AREA(%g1)
1296	ldx	[%g1 + MMFSA_D_TYPE], %g4
1297	stxa	%g4, [%g3 + PTL1_DMMU_TYPE] %asi
1298	ldx	[%g1 + MMFSA_D_ADDR], %g4
1299	stxa	%g4, [%g3 + PTL1_DMMU_ADDR] %asi
1300	ldx	[%g1 + MMFSA_D_CTX], %g4
1301	stxa	%g4, [%g3 + PTL1_DMMU_CTX] %asi
1302	ldx	[%g1 + MMFSA_I_TYPE], %g4
1303	stxa	%g4, [%g3 + PTL1_IMMU_TYPE] %asi
1304	ldx	[%g1 + MMFSA_I_ADDR], %g4
1305	stxa	%g4, [%g3 + PTL1_IMMU_ADDR] %asi
1306	ldx	[%g1 + MMFSA_I_CTX], %g4
1307	stxa	%g4, [%g3 + PTL1_IMMU_CTX] %asi
1308
1309	!
1310	! Save register window state and register windows.
1311	!
1312	rdpr	%cwp, %g1
1313	stba	%g1, [%g3 + PTL1_CWP] %asi
1314	rdpr	%wstate, %g1
1315	stba	%g1, [%g3 + PTL1_WSTATE] %asi
1316	rdpr	%otherwin, %g1
1317	stba	%g1, [%g3 + PTL1_OTHERWIN] %asi
1318	rdpr	%cleanwin, %g1
1319	stba	%g1, [%g3 + PTL1_CLEANWIN] %asi
1320	rdpr	%cansave, %g1
1321	stba	%g1, [%g3 + PTL1_CANSAVE] %asi
1322	rdpr	%canrestore, %g1
1323	stba	%g1, [%g3 + PTL1_CANRESTORE] %asi
1324
1325	PTL1_RESET_RWINDOWS(%g1)
1326	clr	%g1
1327	wrpr	%g1, %cwp
1328	add	%g3, PTL1_RWINDOW, %g4		! %g4 = &ptl1_rwindow[0];
1329
13303:	PTL1_SAVE_WINDOW(%g4)	! <-------------+
1331	inc	%g1				!
1332	cmp	%g1, MAXWIN			!
1333	bgeu,pn	%icc, 5f			!
1334	wrpr	%g1, %cwp			!
1335	rdpr	%cwp, %g2			!
1336	cmp	%g1, %g2			! saturation check
1337	be,pt	%icc, 3b			!
1338	  PTL1_NEXT_WINDOW(%g4)		! ------+
13395:
1340	!
1341	! most crucial CPU state was saved.
1342	! Proceed to go back to TL = 0.
1343	!
1344state_saved:
1345	wrpr	%g0, 1, %tl
1346	wrpr	%g0, 1, %gl
1347	wrpr	%g0, PIL_MAX, %pil
1348	!
1349	PTL1_RESET_RWINDOWS(%g1)
1350	wrpr	%g0, %cwp
1351	wrpr	%g0, %cleanwin
1352	wrpr	%g0, WSTATE_KERN, %wstate
1353	!
1354	! Set pcontext to run kernel.
1355	!
1356	set	MMU_PCONTEXT, %g1
1357	stxa	%g0, [%g1]ASI_MMU_CTX
1358	membar	#Sync
1359
1360	rdpr	%cwp, %g1
1361	set	TSTATE_KERN, %g3
1362	wrpr	%g3, %g1, %tstate
1363	set	ptl1_panic_tl0, %g3
1364	wrpr	%g0, %g3, %tnpc
1365	done					! go to -->-+	TL:1
1366							    !
1367ptl1_panic_tl0:					! ----<-----+	TL:0
1368	CPU_ADDR(%l0, %l1)			! %l0 = cpu[cpuid]
1369	add	%l0, CPU_PTL1, %l1		! %l1 = &CPU->mcpu.ptl1_state
1370	!
1371	! prepare to call panic()
1372	!
1373	ldn	[%l0 + CPU_THREAD], THREAD_REG	! restore %g7
1374	ldn	[%l1 + PTL1_STKTOP], %l2	! %sp = ptl1_stktop
1375	sub	%l2, SA(MINFRAME) + STACK_BIAS, %sp
1376	clr	%fp				! no frame below this window
1377	clr	%i7
1378	!
1379	! enable limited interrupts
1380	!
1381	wrpr	%g0, CLOCK_LEVEL, %pil
1382	wrpr	%g0, PSTATE_KERN, %pstate
1383	!
1384	ba,pt	%xcc, ptl1_panic_handler
1385	  mov	%l1, %o0
1386	/*NOTREACHED*/
1387	SET_SIZE(ptl1_panic)
1388
1389#ifdef	PTL1_PANIC_DEBUG
1390
1391/*
1392 * ptl1_recurse() calls itself a number of times to either set up a known
1393 * stack or to cause a kernel stack overflow. It decrements the arguments
1394 * on each recursion.
1395 * It's called by #ifdef PTL1_PANIC_DEBUG code in startup.c to set the
1396 * registers to a known state to facilitate debugging.
1397 */
1398	ENTRY_NP(ptl1_recurse)
1399	save    %sp, -SA(MINFRAME), %sp
1400
1401	set 	ptl1_recurse_call, %o7
1402	cmp	%o7, %i7			! if ptl1_recurse is called
1403	be,pt  %icc, 0f				! by itself, then skip
1404	  nop					! register initialization
1405
1406	/*
1407	 * Initialize Out Registers to Known Values
1408	 */
1409	set	0x01000, %l0			! %i0 is the ...
1410						! recursion_depth_count
1411	sub	%i0, 1, %o0;
1412	sub 	%i1, 1, %o1;
1413	add	%l0, %o0, %o2;
1414	add	%l0, %o2, %o3;
1415	add	%l0, %o3, %o4;
1416	add	%l0, %o4, %o5;
1417	ba,a	1f
1418	  nop
1419
14200:	/* Outs = Ins - 1 */
1421	sub	%i0, 1, %o0;
1422	sub	%i1, 1, %o1;
1423	sub	%i2, 1, %o2;
1424	sub	%i3, 1, %o3;
1425	sub	%i4, 1, %o4;
1426	sub	%i5, 1, %o5;
1427
1428	/* Locals = Ins + 1 */
14291:	add	%i0, 1, %l0;
1430	add	%i1, 1, %l1;
1431	add	%i2, 1, %l2;
1432	add	%i3, 1, %l3;
1433	add	%i4, 1, %l4;
1434	add	%i5, 1, %l5;
1435
1436	set     0x0100000, %g5
1437	add	%g5, %g0, %g1
1438	add	%g5, %g1, %g2
1439	add	%g5, %g2, %g3
1440	add	%g5, %g3, %g4
1441	add	%g5, %g4, %g5
1442
1443	brz,pn %i1, ptl1_recurse_trap		! if trpp_count == 0) {
1444	  nop					!    trap to ptl1_panic
1445						!
1446	brz,pn %i0, ptl1_recure_exit		! if(depth_count == 0) {
1447	  nop					!    skip recursive call
1448						! }
1449ptl1_recurse_call:
1450	call	ptl1_recurse
1451	  nop
1452
1453ptl1_recure_exit:
1454	ret
1455	restore
1456
1457ptl1_recurse_trap:
1458	ta	PTL1_DEBUG_TRAP; 		! Trap Always to ptl1_panic()
1459	  nop 					! NOTREACHED
1460        SET_SIZE(ptl1_recurse)
1461
1462	/*
1463	 * Asm function to handle a cross trap to call ptl1_panic()
1464	 */
1465	ENTRY_NP(ptl1_panic_xt)
1466	ba	ptl1_panic
1467	  mov	PTL1_BAD_DEBUG, %g1
1468        SET_SIZE(ptl1_panic_xt)
1469
1470#endif	/* PTL1_PANIC_DEBUG */
1471
1472#ifdef	TRAPTRACE
1473
1474	ENTRY_NP(trace_ptr_panic)
1475	!
1476	! freeze the trap trace to disable the assertions.  Otherwise,
1477	! ptl1_panic is likely to be repeatedly called from there.
1478	! %g2 and %g3 are used as scratch registers in ptl1_panic.
1479	!
1480	mov	1, %g3
1481	sethi	%hi(trap_freeze), %g2
1482	st	%g3, [%g2 + %lo(trap_freeze)]
1483	!
1484	! %g1 contains the %pc address where an assertion was failed.
1485	! save it in trap_freeze_pc for a debugging hint if there is
1486	! no value saved in it.
1487	!
1488	set	trap_freeze_pc, %g2
1489	casn	[%g2], %g0, %g1
1490
1491	ba	ptl1_panic
1492	mov	PTL1_BAD_TRACE_PTR, %g1
1493	SET_SIZE(trace_ptr_panic)
1494
1495#endif	/* TRAPTRACE */
1496
1497/*
1498 * The interface for a 32-bit client program that takes over the TBA
1499 * calling the 64-bit romvec OBP.
1500 */
1501
1502	ENTRY(client_handler)
1503	save	%sp, -SA64(MINFRAME64), %sp	! 32 bit frame, 64 bit sized
1504	sethi	%hi(tba_taken_over), %l2
1505	ld	[%l2+%lo(tba_taken_over)], %l3
1506	brz	%l3, 1f				! is the tba_taken_over = 1 ?
1507	rdpr	%wstate, %l5			! save %wstate
1508	andn	%l5, WSTATE_MASK, %l6
1509	wrpr	%l6, WSTATE_KMIX, %wstate
15101:	mov	%i1, %o0
15111:	rdpr	%pstate, %l4			! Get the present pstate value
1512	andn	%l4, PSTATE_AM, %l6
1513	wrpr	%l6, 0, %pstate			! Set PSTATE_AM = 0
1514	jmpl	%i0, %o7			! Call cif handler
1515	nop
1516	wrpr	%l4, 0, %pstate			! restore pstate
1517	brz	%l3, 1f				! is the tba_taken_over = 1
1518	  nop
1519	wrpr	%g0, %l5, %wstate		! restore wstate
15201:	ret					! Return result ...
1521	restore	%o0, %g0, %o0			! delay; result in %o0
1522	SET_SIZE(client_handler)
1523
1524	.seg	".text"
1525bad_hcall_error:
1526	.asciz	"hypervisor call 0x%x returned an unexpected error %d"
1527
1528	/*
1529	 * panic_bad_hcall is called when a hcall returns
1530	 * unexpected error
1531	 * %o0 error number
1532	 * %o1 hcall number
1533	 */
1534
1535	ENTRY(panic_bad_hcall)
1536	mov	%o0, %o2
1537	sethi	%hi(bad_hcall_error), %o0
1538	or	%o0, %lo(bad_hcall_error), %o0
1539	mov	%o7, %o3
1540	call	panic
1541	mov	%o3, %o7
1542	SET_SIZE(panic_bad_hcall)
1543
1544