x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

21 files changed, 10855 insertions, 0 deletions

diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile
new file mode 100644
index 000000000..e9d232561
--- /dev/null
+++ b/arch/sparc/mm/Makefile
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+# Makefile for the linux Sparc-specific parts of the memory manager.
+#
+
+obj-$(CONFIG_SPARC64)   += ultra.o tlb.o tsb.o
+obj-y                   += fault_$(BITS).o
+obj-y                   += init_$(BITS).o
+obj-$(CONFIG_SPARC32)   += srmmu.o iommu.o io-unit.o
+obj-$(CONFIG_SPARC32)   += srmmu_access.o
+obj-$(CONFIG_SPARC32)   += hypersparc.o viking.o tsunami.o swift.o
+obj-$(CONFIG_SPARC32)   += leon_mm.o
+
+# Only used by sparc64
+obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
+
+obj-$(CONFIG_EXECMEM)	+= execmem.o
diff --git a/arch/sparc/mm/execmem.c b/arch/sparc/mm/execmem.c
new file mode 100644
index 000000000..0fac97dd5
--- /dev/null
+++ b/arch/sparc/mm/execmem.c
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/mm.h>
+#include <linux/execmem.h>
+
+static struct execmem_info execmem_info __ro_after_init;
+
+struct execmem_info __init *execmem_arch_setup(void)
+{
+	execmem_info = (struct execmem_info){
+		.ranges = {
+			[EXECMEM_DEFAULT] = {
+				.start	= MODULES_VADDR,
+				.end	= MODULES_END,
+				.pgprot	= PAGE_KERNEL,
+				.alignment = 1,
+			},
+		},
+	};
+
+	return &execmem_info;
+}
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
new file mode 100644
index 000000000..86a831ebd
--- /dev/null
+++ b/arch/sparc/mm/fault_32.c
@@ -0,0 +1,382 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * fault.c:  Page fault handlers for the Sparc.
+ *
+ * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
+ * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
+ * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
+ */
+
+#include <asm/head.h>
+
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/sched.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/threads.h>
+#include <linux/kernel.h>
+#include <linux/signal.h>
+#include <linux/mm.h>
+#include <linux/smp.h>
+#include <linux/perf_event.h>
+#include <linux/interrupt.h>
+#include <linux/kdebug.h>
+#include <linux/uaccess.h>
+#include <linux/extable.h>
+
+#include <asm/page.h>
+#include <asm/openprom.h>
+#include <asm/oplib.h>
+#include <asm/setup.h>
+#include <asm/smp.h>
+#include <asm/traps.h>
+
+#include "mm_32.h"
+
+int show_unhandled_signals = 1;
+
+static void __noreturn unhandled_fault(unsigned long address,
+				       struct task_struct *tsk,
+				       struct pt_regs *regs)
+{
+	if ((unsigned long) address < PAGE_SIZE) {
+		printk(KERN_ALERT
+		    "Unable to handle kernel NULL pointer dereference\n");
+	} else {
+		printk(KERN_ALERT "Unable to handle kernel paging request at virtual address %08lx\n",
+		       address);
+	}
+	printk(KERN_ALERT "tsk->{mm,active_mm}->context = %08lx\n",
+		(tsk->mm ? tsk->mm->context : tsk->active_mm->context));
+	printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %08lx\n",
+		(tsk->mm ? (unsigned long) tsk->mm->pgd :
+			(unsigned long) tsk->active_mm->pgd));
+	die_if_kernel("Oops", regs);
+}
+
+static inline void
+show_signal_msg(struct pt_regs *regs, int sig, int code,
+		unsigned long address, struct task_struct *tsk)
+{
+	if (!unhandled_signal(tsk, sig))
+		return;
+
+	if (!printk_ratelimit())
+		return;
+
+	printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x",
+	       task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
+	       tsk->comm, task_pid_nr(tsk), address,
+	       (void *)regs->pc, (void *)regs->u_regs[UREG_I7],
+	       (void *)regs->u_regs[UREG_FP], code);
+
+	print_vma_addr(KERN_CONT " in ", regs->pc);
+
+	printk(KERN_CONT "\n");
+}
+
+static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs,
+			       unsigned long addr)
+{
+	if (unlikely(show_unhandled_signals))
+		show_signal_msg(regs, sig, code,
+				addr, current);
+
+	force_sig_fault(sig, code, (void __user *) addr);
+}
+
+static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault)
+{
+	unsigned int insn;
+
+	if (text_fault)
+		return regs->pc;
+
+	if (regs->psr & PSR_PS)
+		insn = *(unsigned int *) regs->pc;
+	else
+		__get_user(insn, (unsigned int *) regs->pc);
+
+	return safe_compute_effective_address(regs, insn);
+}
+
+static noinline void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
+				      int text_fault)
+{
+	unsigned long addr = compute_si_addr(regs, text_fault);
+
+	__do_fault_siginfo(code, sig, regs, addr);
+}
+
+asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
+			       unsigned long address)
+{
+	struct vm_area_struct *vma;
+	struct task_struct *tsk = current;
+	struct mm_struct *mm = tsk->mm;
+	int from_user = !(regs->psr & PSR_PS);
+	int code;
+	vm_fault_t fault;
+	unsigned int flags = FAULT_FLAG_DEFAULT;
+
+	if (text_fault)
+		address = regs->pc;
+
+	/*
+	 * We fault-in kernel-space virtual memory on-demand. The
+	 * 'reference' page table is init_mm.pgd.
+	 *
+	 * NOTE! We MUST NOT take any locks for this case. We may
+	 * be in an interrupt or a critical region, and should
+	 * only copy the information from the master page table,
+	 * nothing more.
+	 */
+	code = SEGV_MAPERR;
+	if (address >= TASK_SIZE)
+		goto vmalloc_fault;
+
+	/*
+	 * If we're in an interrupt or have no user
+	 * context, we must not take the fault..
+	 */
+	if (pagefault_disabled() || !mm)
+		goto no_context;
+
+	if (!from_user && address >= PAGE_OFFSET)
+		goto no_context;
+
+	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
+
+retry:
+	vma = lock_mm_and_find_vma(mm, address, regs);
+	if (!vma)
+		goto bad_area_nosemaphore;
+	/*
+	 * Ok, we have a good vm_area for this memory access, so
+	 * we can handle it..
+	 */
+	code = SEGV_ACCERR;
+	if (write) {
+		if (!(vma->vm_flags & VM_WRITE))
+			goto bad_area;
+	} else {
+		/* Allow reads even for write-only mappings */
+		if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
+			goto bad_area;
+	}
+
+	if (from_user)
+		flags |= FAULT_FLAG_USER;
+	if (write)
+		flags |= FAULT_FLAG_WRITE;
+
+	/*
+	 * If for any reason at all we couldn't handle the fault,
+	 * make sure we exit gracefully rather than endlessly redo
+	 * the fault.
+	 */
+	fault = handle_mm_fault(vma, address, flags, regs);
+
+	if (fault_signal_pending(fault, regs)) {
+		if (!from_user)
+			goto no_context;
+		return;
+	}
+
+	/* The fault is fully completed (including releasing mmap lock) */
+	if (fault & VM_FAULT_COMPLETED)
+		return;
+
+	if (unlikely(fault & VM_FAULT_ERROR)) {
+		if (fault & VM_FAULT_OOM)
+			goto out_of_memory;
+		else if (fault & VM_FAULT_SIGSEGV)
+			goto bad_area;
+		else if (fault & VM_FAULT_SIGBUS)
+			goto do_sigbus;
+		BUG();
+	}
+
+	if (fault & VM_FAULT_RETRY) {
+		flags |= FAULT_FLAG_TRIED;
+
+		/* No need to mmap_read_unlock(mm) as we would
+		 * have already released it in __lock_page_or_retry
+		 * in mm/filemap.c.
+		 */
+
+		goto retry;
+	}
+
+	mmap_read_unlock(mm);
+	return;
+
+	/*
+	 * Something tried to access memory that isn't in our memory map..
+	 * Fix it, but check if it's kernel or user first..
+	 */
+bad_area:
+	mmap_read_unlock(mm);
+
+bad_area_nosemaphore:
+	/* User mode accesses just cause a SIGSEGV */
+	if (from_user) {
+		do_fault_siginfo(code, SIGSEGV, regs, text_fault);
+		return;
+	}
+
+	/* Is this in ex_table? */
+no_context:
+	if (!from_user) {
+		const struct exception_table_entry *entry;
+
+		entry = search_exception_tables(regs->pc);
+#ifdef DEBUG_EXCEPTIONS
+		printk("Exception: PC<%08lx> faddr<%08lx>\n",
+		       regs->pc, address);
+		printk("EX_TABLE: insn<%08lx> fixup<%08x>\n",
+			regs->pc, entry->fixup);
+#endif
+		regs->pc = entry->fixup;
+		regs->npc = regs->pc + 4;
+		return;
+	}
+
+	unhandled_fault(address, tsk, regs);
+
+/*
+ * We ran out of memory, or some other thing happened to us that made
+ * us unable to handle the page fault gracefully.
+ */
+out_of_memory:
+	mmap_read_unlock(mm);
+	if (from_user) {
+		pagefault_out_of_memory();
+		return;
+	}
+	goto no_context;
+
+do_sigbus:
+	mmap_read_unlock(mm);
+	do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, text_fault);
+	if (!from_user)
+		goto no_context;
+
+vmalloc_fault:
+	{
+		/*
+		 * Synchronize this task's top level page-table
+		 * with the 'reference' page table.
+		 */
+		int offset = pgd_index(address);
+		pgd_t *pgd, *pgd_k;
+		p4d_t *p4d, *p4d_k;
+		pud_t *pud, *pud_k;
+		pmd_t *pmd, *pmd_k;
+
+		pgd = tsk->active_mm->pgd + offset;
+		pgd_k = init_mm.pgd + offset;
+
+		if (!pgd_present(*pgd)) {
+			if (!pgd_present(*pgd_k))
+				goto bad_area_nosemaphore;
+			pgd_val(*pgd) = pgd_val(*pgd_k);
+			return;
+		}
+
+		p4d = p4d_offset(pgd, address);
+		pud = pud_offset(p4d, address);
+		pmd = pmd_offset(pud, address);
+
+		p4d_k = p4d_offset(pgd_k, address);
+		pud_k = pud_offset(p4d_k, address);
+		pmd_k = pmd_offset(pud_k, address);
+
+		if (pmd_present(*pmd) || !pmd_present(*pmd_k))
+			goto bad_area_nosemaphore;
+
+		*pmd = *pmd_k;
+		return;
+	}
+}
+
+/* This always deals with user addresses. */
+static void force_user_fault(unsigned long address, int write)
+{
+	struct vm_area_struct *vma;
+	struct task_struct *tsk = current;
+	struct mm_struct *mm = tsk->mm;
+	unsigned int flags = FAULT_FLAG_USER;
+	int code;
+
+	code = SEGV_MAPERR;
+
+	vma = lock_mm_and_find_vma(mm, address, NULL);
+	if (!vma)
+		goto bad_area_nosemaphore;
+	code = SEGV_ACCERR;
+	if (write) {
+		if (!(vma->vm_flags & VM_WRITE))
+			goto bad_area;
+		flags |= FAULT_FLAG_WRITE;
+	} else {
+		if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
+			goto bad_area;
+	}
+	switch (handle_mm_fault(vma, address, flags, NULL)) {
+	case VM_FAULT_SIGBUS:
+	case VM_FAULT_OOM:
+		goto do_sigbus;
+	}
+	mmap_read_unlock(mm);
+	return;
+bad_area:
+	mmap_read_unlock(mm);
+bad_area_nosemaphore:
+	__do_fault_siginfo(code, SIGSEGV, tsk->thread.kregs, address);
+	return;
+
+do_sigbus:
+	mmap_read_unlock(mm);
+	__do_fault_siginfo(BUS_ADRERR, SIGBUS, tsk->thread.kregs, address);
+}
+
+static void check_stack_aligned(unsigned long sp)
+{
+	if (sp & 0x7UL)
+		force_sig(SIGILL);
+}
+
+void window_overflow_fault(void)
+{
+	unsigned long sp;
+
+	sp = current_thread_info()->rwbuf_stkptrs[0];
+	if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK))
+		force_user_fault(sp + 0x38, 1);
+	force_user_fault(sp, 1);
+
+	check_stack_aligned(sp);
+}
+
+void window_underflow_fault(unsigned long sp)
+{
+	if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK))
+		force_user_fault(sp + 0x38, 0);
+	force_user_fault(sp, 0);
+
+	check_stack_aligned(sp);
+}
+
+void window_ret_fault(struct pt_regs *regs)
+{
+	unsigned long sp;
+
+	sp = regs->u_regs[UREG_FP];
+	if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK))
+		force_user_fault(sp + 0x38, 0);
+	force_user_fault(sp, 0);
+
+	check_stack_aligned(sp);
+}
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
new file mode 100644
index 000000000..e326caf70
--- /dev/null
+++ b/arch/sparc/mm/fault_64.c
@@ -0,0 +1,532 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc.
+ *
+ * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net)
+ * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ */
+
+#include <asm/head.h>
+
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/sched.h>
+#include <linux/sched/debug.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/signal.h>
+#include <linux/mm.h>
+#include <linux/extable.h>
+#include <linux/init.h>
+#include <linux/perf_event.h>
+#include <linux/interrupt.h>
+#include <linux/kprobes.h>
+#include <linux/kdebug.h>
+#include <linux/percpu.h>
+#include <linux/context_tracking.h>
+#include <linux/uaccess.h>
+
+#include <asm/page.h>
+#include <asm/openprom.h>
+#include <asm/oplib.h>
+#include <asm/asi.h>
+#include <asm/lsu.h>
+#include <asm/sections.h>
+#include <asm/mmu_context.h>
+#include <asm/setup.h>
+
+int show_unhandled_signals = 1;
+
+static void __kprobes unhandled_fault(unsigned long address,
+				      struct task_struct *tsk,
+				      struct pt_regs *regs)
+{
+	if ((unsigned long) address < PAGE_SIZE) {
+		printk(KERN_ALERT "Unable to handle kernel NULL "
+		       "pointer dereference\n");
+	} else {
+		printk(KERN_ALERT "Unable to handle kernel paging request "
+		       "at virtual address %016lx\n", (unsigned long)address);
+	}
+	printk(KERN_ALERT "tsk->{mm,active_mm}->context = %016lx\n",
+	       (tsk->mm ?
+		CTX_HWBITS(tsk->mm->context) :
+		CTX_HWBITS(tsk->active_mm->context)));
+	printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %016lx\n",
+	       (tsk->mm ? (unsigned long) tsk->mm->pgd :
+		          (unsigned long) tsk->active_mm->pgd));
+	die_if_kernel("Oops", regs);
+}
+
+static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
+{
+	printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n",
+	       regs->tpc);
+	printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]);
+	printk("OOPS: RPC <%pS>\n", (void *) regs->u_regs[15]);
+	printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr);
+	dump_stack();
+	unhandled_fault(regs->tpc, current, regs);
+}
+
+/*
+ * We now make sure that mmap_lock is held in all paths that call
+ * this. Additionally, to prevent kswapd from ripping ptes from
+ * under us, raise interrupts around the time that we look at the
+ * pte, kswapd will have to wait to get his smp ipi response from
+ * us. vmtruncate likewise. This saves us having to get pte lock.
+ */
+static unsigned int get_user_insn(unsigned long tpc)
+{
+	pgd_t *pgdp = pgd_offset(current->mm, tpc);
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep, pte;
+	unsigned long pa;
+	u32 insn = 0;
+
+	if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
+		goto out;
+	p4dp = p4d_offset(pgdp, tpc);
+	if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp)))
+		goto out;
+	pudp = pud_offset(p4dp, tpc);
+	if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
+		goto out;
+
+	/* This disables preemption for us as well. */
+	local_irq_disable();
+
+	pmdp = pmd_offset(pudp, tpc);
+again:
+	if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp)))
+		goto out_irq_enable;
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	if (is_hugetlb_pmd(*pmdp)) {
+		pa  = pmd_pfn(*pmdp) << PAGE_SHIFT;
+		pa += tpc & ~HPAGE_MASK;
+
+		/* Use phys bypass so we don't pollute dtlb/dcache. */
+		__asm__ __volatile__("lduwa [%1] %2, %0"
+				     : "=r" (insn)
+				     : "r" (pa), "i" (ASI_PHYS_USE_EC));
+	} else
+#endif
+	{
+		ptep = pte_offset_map(pmdp, tpc);
+		if (!ptep)
+			goto again;
+		pte = *ptep;
+		if (pte_present(pte)) {
+			pa  = (pte_pfn(pte) << PAGE_SHIFT);
+			pa += (tpc & ~PAGE_MASK);
+
+			/* Use phys bypass so we don't pollute dtlb/dcache. */
+			__asm__ __volatile__("lduwa [%1] %2, %0"
+					     : "=r" (insn)
+					     : "r" (pa), "i" (ASI_PHYS_USE_EC));
+		}
+		pte_unmap(ptep);
+	}
+out_irq_enable:
+	local_irq_enable();
+out:
+	return insn;
+}
+
+static inline void
+show_signal_msg(struct pt_regs *regs, int sig, int code,
+		unsigned long address, struct task_struct *tsk)
+{
+	if (!unhandled_signal(tsk, sig))
+		return;
+
+	if (!printk_ratelimit())
+		return;
+
+	printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x",
+	       task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
+	       tsk->comm, task_pid_nr(tsk), address,
+	       (void *)regs->tpc, (void *)regs->u_regs[UREG_I7],
+	       (void *)regs->u_regs[UREG_FP], code);
+
+	print_vma_addr(KERN_CONT " in ", regs->tpc);
+
+	printk(KERN_CONT "\n");
+}
+
+static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
+			     unsigned long fault_addr, unsigned int insn,
+			     int fault_code)
+{
+	unsigned long addr;
+
+	if (fault_code & FAULT_CODE_ITLB) {
+		addr = regs->tpc;
+	} else {
+		/* If we were able to probe the faulting instruction, use it
+		 * to compute a precise fault address.  Otherwise use the fault
+		 * time provided address which may only have page granularity.
+		 */
+		if (insn)
+			addr = compute_effective_address(regs, insn, 0);
+		else
+			addr = fault_addr;
+	}
+
+	if (unlikely(show_unhandled_signals))
+		show_signal_msg(regs, sig, code, addr, current);
+
+	force_sig_fault(sig, code, (void __user *) addr);
+}
+
+static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
+{
+	if (!insn) {
+		if (!regs->tpc || (regs->tpc & 0x3))
+			return 0;
+		if (regs->tstate & TSTATE_PRIV) {
+			insn = *(unsigned int *) regs->tpc;
+		} else {
+			insn = get_user_insn(regs->tpc);
+		}
+	}
+	return insn;
+}
+
+static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code,
+				      int fault_code, unsigned int insn,
+				      unsigned long address)
+{
+	unsigned char asi = ASI_P;
+ 
+	if ((!insn) && (regs->tstate & TSTATE_PRIV))
+		goto cannot_handle;
+
+	/* If user insn could be read (thus insn is zero), that
+	 * is fine.  We will just gun down the process with a signal
+	 * in that case.
+	 */
+
+	if (!(fault_code & (FAULT_CODE_WRITE|FAULT_CODE_ITLB)) &&
+	    (insn & 0xc0800000) == 0xc0800000) {
+		if (insn & 0x2000)
+			asi = (regs->tstate >> 24);
+		else
+			asi = (insn >> 5);
+		if ((asi & 0xf2) == 0x82) {
+			if (insn & 0x1000000) {
+				handle_ldf_stq(insn, regs);
+			} else {
+				/* This was a non-faulting load. Just clear the
+				 * destination register(s) and continue with the next
+				 * instruction. -jj
+				 */
+				handle_ld_nf(insn, regs);
+			}
+			return;
+		}
+	}
+		
+	/* Is this in ex_table? */
+	if (regs->tstate & TSTATE_PRIV) {
+		const struct exception_table_entry *entry;
+
+		entry = search_exception_tables(regs->tpc);
+		if (entry) {
+			regs->tpc = entry->fixup;
+			regs->tnpc = regs->tpc + 4;
+			return;
+		}
+	} else {
+		/* The si_code was set to make clear whether
+		 * this was a SEGV_MAPERR or SEGV_ACCERR fault.
+		 */
+		do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code);
+		return;
+	}
+
+cannot_handle:
+	unhandled_fault (address, current, regs);
+}
+
+static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs)
+{
+	static int times;
+
+	if (times++ < 10)
+		printk(KERN_ERR "FAULT[%s:%d]: 32-bit process reports "
+		       "64-bit TPC [%lx]\n",
+		       current->comm, current->pid,
+		       regs->tpc);
+	show_regs(regs);
+}
+
+asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
+{
+	enum ctx_state prev_state = exception_enter();
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma;
+	unsigned int insn = 0;
+	int si_code, fault_code;
+	vm_fault_t fault;
+	unsigned long address, mm_rss;
+	unsigned int flags = FAULT_FLAG_DEFAULT;
+
+	fault_code = get_thread_fault_code();
+
+	if (kprobe_page_fault(regs, 0))
+		goto exit_exception;
+
+	si_code = SEGV_MAPERR;
+	address = current_thread_info()->fault_address;
+
+	if ((fault_code & FAULT_CODE_ITLB) &&
+	    (fault_code & FAULT_CODE_DTLB))
+		BUG();
+
+	if (test_thread_flag(TIF_32BIT)) {
+		if (!(regs->tstate & TSTATE_PRIV)) {
+			if (unlikely((regs->tpc >> 32) != 0)) {
+				bogus_32bit_fault_tpc(regs);
+				goto intr_or_no_mm;
+			}
+		}
+		if (unlikely((address >> 32) != 0))
+			goto intr_or_no_mm;
+	}
+
+	if (regs->tstate & TSTATE_PRIV) {
+		unsigned long tpc = regs->tpc;
+
+		/* Sanity check the PC. */
+		if ((tpc >= KERNBASE && tpc < (unsigned long) __init_end) ||
+		    (tpc >= MODULES_VADDR && tpc < MODULES_END)) {
+			/* Valid, no problems... */
+		} else {
+			bad_kernel_pc(regs, address);
+			goto exit_exception;
+		}
+	} else
+		flags |= FAULT_FLAG_USER;
+
+	/*
+	 * If we're in an interrupt or have no user
+	 * context, we must not take the fault..
+	 */
+	if (faulthandler_disabled() || !mm)
+		goto intr_or_no_mm;
+
+	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
+
+	if (!mmap_read_trylock(mm)) {
+		if ((regs->tstate & TSTATE_PRIV) &&
+		    !search_exception_tables(regs->tpc)) {
+			insn = get_fault_insn(regs, insn);
+			goto handle_kernel_fault;
+		}
+
+retry:
+		mmap_read_lock(mm);
+	}
+
+	if (fault_code & FAULT_CODE_BAD_RA)
+		goto do_sigbus;
+
+	vma = find_vma(mm, address);
+	if (!vma)
+		goto bad_area;
+
+	/* Pure DTLB misses do not tell us whether the fault causing
+	 * load/store/atomic was a write or not, it only says that there
+	 * was no match.  So in such a case we (carefully) read the
+	 * instruction to try and figure this out.  It's an optimization
+	 * so it's ok if we can't do this.
+	 *
+	 * Special hack, window spill/fill knows the exact fault type.
+	 */
+	if (((fault_code &
+	      (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) &&
+	    (vma->vm_flags & VM_WRITE) != 0) {
+		insn = get_fault_insn(regs, 0);
+		if (!insn)
+			goto continue_fault;
+		/* All loads, stores and atomics have bits 30 and 31 both set
+		 * in the instruction.  Bit 21 is set in all stores, but we
+		 * have to avoid prefetches which also have bit 21 set.
+		 */
+		if ((insn & 0xc0200000) == 0xc0200000 &&
+		    (insn & 0x01780000) != 0x01680000) {
+			/* Don't bother updating thread struct value,
+			 * because update_mmu_cache only cares which tlb
+			 * the access came from.
+			 */
+			fault_code |= FAULT_CODE_WRITE;
+		}
+	}
+continue_fault:
+
+	if (vma->vm_start <= address)
+		goto good_area;
+	if (!(vma->vm_flags & VM_GROWSDOWN))
+		goto bad_area;
+	if (!(fault_code & FAULT_CODE_WRITE)) {
+		/* Non-faulting loads shouldn't expand stack. */
+		insn = get_fault_insn(regs, insn);
+		if ((insn & 0xc0800000) == 0xc0800000) {
+			unsigned char asi;
+
+			if (insn & 0x2000)
+				asi = (regs->tstate >> 24);
+			else
+				asi = (insn >> 5);
+			if ((asi & 0xf2) == 0x82)
+				goto bad_area;
+		}
+	}
+	vma = expand_stack(mm, address);
+	if (!vma)
+		goto bad_area_nosemaphore;
+	/*
+	 * Ok, we have a good vm_area for this memory access, so
+	 * we can handle it..
+	 */
+good_area:
+	si_code = SEGV_ACCERR;
+
+	/* If we took a ITLB miss on a non-executable page, catch
+	 * that here.
+	 */
+	if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) {
+		WARN(address != regs->tpc,
+		     "address (%lx) != regs->tpc (%lx)\n", address, regs->tpc);
+		WARN_ON(regs->tstate & TSTATE_PRIV);
+		goto bad_area;
+	}
+
+	if (fault_code & FAULT_CODE_WRITE) {
+		if (!(vma->vm_flags & VM_WRITE))
+			goto bad_area;
+
+		/* Spitfire has an icache which does not snoop
+		 * processor stores.  Later processors do...
+		 */
+		if (tlb_type == spitfire &&
+		    (vma->vm_flags & VM_EXEC) != 0 &&
+		    vma->vm_file != NULL)
+			set_thread_fault_code(fault_code |
+					      FAULT_CODE_BLKCOMMIT);
+
+		flags |= FAULT_FLAG_WRITE;
+	} else {
+		/* Allow reads even for write-only mappings */
+		if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
+			goto bad_area;
+	}
+
+	fault = handle_mm_fault(vma, address, flags, regs);
+
+	if (fault_signal_pending(fault, regs)) {
+		if (regs->tstate & TSTATE_PRIV) {
+			insn = get_fault_insn(regs, insn);
+			goto handle_kernel_fault;
+		}
+		goto exit_exception;
+	}
+
+	/* The fault is fully completed (including releasing mmap lock) */
+	if (fault & VM_FAULT_COMPLETED)
+		goto lock_released;
+
+	if (unlikely(fault & VM_FAULT_ERROR)) {
+		if (fault & VM_FAULT_OOM)
+			goto out_of_memory;
+		else if (fault & VM_FAULT_SIGSEGV)
+			goto bad_area;
+		else if (fault & VM_FAULT_SIGBUS)
+			goto do_sigbus;
+		BUG();
+	}
+
+	if (fault & VM_FAULT_RETRY) {
+		flags |= FAULT_FLAG_TRIED;
+
+		/* No need to mmap_read_unlock(mm) as we would
+		 * have already released it in __lock_page_or_retry
+		 * in mm/filemap.c.
+		 */
+
+		goto retry;
+	}
+	mmap_read_unlock(mm);
+
+lock_released:
+	mm_rss = get_mm_rss(mm);
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE));
+#endif
+	if (unlikely(mm_rss >
+		     mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
+		tsb_grow(mm, MM_TSB_BASE, mm_rss);
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count;
+	mm_rss *= REAL_HPAGE_PER_HPAGE;
+	if (unlikely(mm_rss >
+		     mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
+		if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
+			tsb_grow(mm, MM_TSB_HUGE, mm_rss);
+		else
+			hugetlb_setup(regs);
+
+	}
+#endif
+exit_exception:
+	exception_exit(prev_state);
+	return;
+
+	/*
+	 * Something tried to access memory that isn't in our memory map..
+	 * Fix it, but check if it's kernel or user first..
+	 */
+bad_area:
+	mmap_read_unlock(mm);
+bad_area_nosemaphore:
+	insn = get_fault_insn(regs, insn);
+
+handle_kernel_fault:
+	do_kernel_fault(regs, si_code, fault_code, insn, address);
+	goto exit_exception;
+
+/*
+ * We ran out of memory, or some other thing happened to us that made
+ * us unable to handle the page fault gracefully.
+ */
+out_of_memory:
+	insn = get_fault_insn(regs, insn);
+	mmap_read_unlock(mm);
+	if (!(regs->tstate & TSTATE_PRIV)) {
+		pagefault_out_of_memory();
+		goto exit_exception;
+	}
+	goto handle_kernel_fault;
+
+intr_or_no_mm:
+	insn = get_fault_insn(regs, 0);
+	goto handle_kernel_fault;
+
+do_sigbus:
+	insn = get_fault_insn(regs, insn);
+	mmap_read_unlock(mm);
+
+	/*
+	 * Send a sigbus, regardless of whether we were in kernel
+	 * or user mode.
+	 */
+	do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code);
+
+	/* Kernel mode? Handle exceptions or die */
+	if (regs->tstate & TSTATE_PRIV)
+		goto handle_kernel_fault;
+}
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
new file mode 100644
index 000000000..4652e8686
--- /dev/null
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -0,0 +1,317 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SPARC64 Huge TLB page support.
+ *
+ * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
+ */
+
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/sched/mm.h>
+#include <linux/hugetlb.h>
+#include <linux/pagemap.h>
+#include <linux/sysctl.h>
+
+#include <asm/mman.h>
+#include <asm/pgalloc.h>
+#include <asm/tlb.h>
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+#include <asm/mmu_context.h>
+
+
+static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
+{
+	unsigned long hugepage_size = _PAGE_SZ4MB_4U;
+
+	pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4U;
+
+	switch (shift) {
+	case HPAGE_256MB_SHIFT:
+		hugepage_size = _PAGE_SZ256MB_4U;
+		pte_val(entry) |= _PAGE_PMD_HUGE;
+		break;
+	case HPAGE_SHIFT:
+		pte_val(entry) |= _PAGE_PMD_HUGE;
+		break;
+	case HPAGE_64K_SHIFT:
+		hugepage_size = _PAGE_SZ64K_4U;
+		break;
+	default:
+		WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
+	}
+
+	pte_val(entry) = pte_val(entry) | hugepage_size;
+	return entry;
+}
+
+static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
+{
+	unsigned long hugepage_size = _PAGE_SZ4MB_4V;
+
+	pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
+
+	switch (shift) {
+	case HPAGE_16GB_SHIFT:
+		hugepage_size = _PAGE_SZ16GB_4V;
+		pte_val(entry) |= _PAGE_PUD_HUGE;
+		break;
+	case HPAGE_2GB_SHIFT:
+		hugepage_size = _PAGE_SZ2GB_4V;
+		pte_val(entry) |= _PAGE_PMD_HUGE;
+		break;
+	case HPAGE_256MB_SHIFT:
+		hugepage_size = _PAGE_SZ256MB_4V;
+		pte_val(entry) |= _PAGE_PMD_HUGE;
+		break;
+	case HPAGE_SHIFT:
+		pte_val(entry) |= _PAGE_PMD_HUGE;
+		break;
+	case HPAGE_64K_SHIFT:
+		hugepage_size = _PAGE_SZ64K_4V;
+		break;
+	default:
+		WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
+	}
+
+	pte_val(entry) = pte_val(entry) | hugepage_size;
+	return entry;
+}
+
+static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
+{
+	if (tlb_type == hypervisor)
+		return sun4v_hugepage_shift_to_tte(entry, shift);
+	else
+		return sun4u_hugepage_shift_to_tte(entry, shift);
+}
+
+pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
+{
+	pte_t pte;
+
+	entry = pte_mkhuge(entry);
+	pte = hugepage_shift_to_tte(entry, shift);
+
+#ifdef CONFIG_SPARC64
+	/* If this vma has ADI enabled on it, turn on TTE.mcd
+	 */
+	if (flags & VM_SPARC_ADI)
+		return pte_mkmcd(pte);
+	else
+		return pte_mknotmcd(pte);
+#else
+	return pte;
+#endif
+}
+
+static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
+{
+	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
+	unsigned int shift;
+
+	switch (tte_szbits) {
+	case _PAGE_SZ16GB_4V:
+		shift = HPAGE_16GB_SHIFT;
+		break;
+	case _PAGE_SZ2GB_4V:
+		shift = HPAGE_2GB_SHIFT;
+		break;
+	case _PAGE_SZ256MB_4V:
+		shift = HPAGE_256MB_SHIFT;
+		break;
+	case _PAGE_SZ4MB_4V:
+		shift = REAL_HPAGE_SHIFT;
+		break;
+	case _PAGE_SZ64K_4V:
+		shift = HPAGE_64K_SHIFT;
+		break;
+	default:
+		shift = PAGE_SHIFT;
+		break;
+	}
+	return shift;
+}
+
+static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
+{
+	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
+	unsigned int shift;
+
+	switch (tte_szbits) {
+	case _PAGE_SZ256MB_4U:
+		shift = HPAGE_256MB_SHIFT;
+		break;
+	case _PAGE_SZ4MB_4U:
+		shift = REAL_HPAGE_SHIFT;
+		break;
+	case _PAGE_SZ64K_4U:
+		shift = HPAGE_64K_SHIFT;
+		break;
+	default:
+		shift = PAGE_SHIFT;
+		break;
+	}
+	return shift;
+}
+
+static unsigned long tte_to_shift(pte_t entry)
+{
+	if (tlb_type == hypervisor)
+		return sun4v_huge_tte_to_shift(entry);
+
+	return sun4u_huge_tte_to_shift(entry);
+}
+
+static unsigned int huge_tte_to_shift(pte_t entry)
+{
+	unsigned long shift = tte_to_shift(entry);
+
+	if (shift == PAGE_SHIFT)
+		WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
+			  pte_val(entry));
+
+	return shift;
+}
+
+static unsigned long huge_tte_to_size(pte_t pte)
+{
+	unsigned long size = 1UL << huge_tte_to_shift(pte);
+
+	if (size == REAL_HPAGE_SIZE)
+		size = HPAGE_SIZE;
+	return size;
+}
+
+unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); }
+unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); }
+unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); }
+
+pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
+			unsigned long addr, unsigned long sz)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+
+	pgd = pgd_offset(mm, addr);
+	p4d = p4d_offset(pgd, addr);
+	pud = pud_alloc(mm, p4d, addr);
+	if (!pud)
+		return NULL;
+	if (sz >= PUD_SIZE)
+		return (pte_t *)pud;
+	pmd = pmd_alloc(mm, pud, addr);
+	if (!pmd)
+		return NULL;
+	if (sz >= PMD_SIZE)
+		return (pte_t *)pmd;
+	return pte_alloc_huge(mm, pmd, addr);
+}
+
+pte_t *huge_pte_offset(struct mm_struct *mm,
+		       unsigned long addr, unsigned long sz)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+
+	pgd = pgd_offset(mm, addr);
+	if (pgd_none(*pgd))
+		return NULL;
+	p4d = p4d_offset(pgd, addr);
+	if (p4d_none(*p4d))
+		return NULL;
+	pud = pud_offset(p4d, addr);
+	if (pud_none(*pud))
+		return NULL;
+	if (is_hugetlb_pud(*pud))
+		return (pte_t *)pud;
+	pmd = pmd_offset(pud, addr);
+	if (pmd_none(*pmd))
+		return NULL;
+	if (is_hugetlb_pmd(*pmd))
+		return (pte_t *)pmd;
+	return pte_offset_huge(pmd, addr);
+}
+
+void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
+		     pte_t *ptep, pte_t entry)
+{
+	unsigned int nptes, orig_shift, shift;
+	unsigned long i, size;
+	pte_t orig;
+
+	size = huge_tte_to_size(entry);
+
+	shift = PAGE_SHIFT;
+	if (size >= PUD_SIZE)
+		shift = PUD_SHIFT;
+	else if (size >= PMD_SIZE)
+		shift = PMD_SHIFT;
+	else
+		shift = PAGE_SHIFT;
+
+	nptes = size >> shift;
+
+	if (!pte_present(*ptep) && pte_present(entry))
+		mm->context.hugetlb_pte_count += nptes;
+
+	addr &= ~(size - 1);
+	orig = *ptep;
+	orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
+
+	for (i = 0; i < nptes; i++)
+		ptep[i] = __pte(pte_val(entry) + (i << shift));
+
+	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
+	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
+	if (size == HPAGE_SIZE)
+		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
+				    orig_shift);
+}
+
+void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
+		     pte_t *ptep, pte_t entry, unsigned long sz)
+{
+	__set_huge_pte_at(mm, addr, ptep, entry);
+}
+
+pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
+			      pte_t *ptep, unsigned long sz)
+{
+	unsigned int i, nptes, orig_shift, shift;
+	unsigned long size;
+	pte_t entry;
+
+	entry = *ptep;
+	size = huge_tte_to_size(entry);
+
+	shift = PAGE_SHIFT;
+	if (size >= PUD_SIZE)
+		shift = PUD_SHIFT;
+	else if (size >= PMD_SIZE)
+		shift = PMD_SHIFT;
+	else
+		shift = PAGE_SHIFT;
+
+	nptes = size >> shift;
+	orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
+
+	if (pte_present(entry))
+		mm->context.hugetlb_pte_count -= nptes;
+
+	addr &= ~(size - 1);
+	for (i = 0; i < nptes; i++)
+		ptep[i] = __pte(0UL);
+
+	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
+	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
+	if (size == HPAGE_SIZE)
+		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
+				    orig_shift);
+
+	return entry;
+}
diff --git a/arch/sparc/mm/hypersparc.S b/arch/sparc/mm/hypersparc.S
new file mode 100644
index 000000000..6c2521e85
--- /dev/null
+++ b/arch/sparc/mm/hypersparc.S
@@ -0,0 +1,414 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * hypersparc.S: High speed Hypersparc mmu/cache operations.
+ *
+ * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
+ */
+
+#include <asm/ptrace.h>
+#include <asm/psr.h>
+#include <asm/asm-offsets.h>
+#include <asm/asi.h>
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <asm/pgtsrmmu.h>
+#include <linux/init.h>
+
+	.text
+	.align	4
+
+	.globl	hypersparc_flush_cache_all, hypersparc_flush_cache_mm
+	.globl	hypersparc_flush_cache_range, hypersparc_flush_cache_page
+	.globl	hypersparc_flush_page_to_ram
+	.globl	hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
+	.globl	hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
+	.globl	hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
+
+hypersparc_flush_cache_all:
+	WINDOW_FLUSH(%g4, %g5)
+	sethi	%hi(vac_cache_size), %g4
+	ld	[%g4 + %lo(vac_cache_size)], %g5
+	sethi	%hi(vac_line_size), %g1
+	ld	[%g1 + %lo(vac_line_size)], %g2
+1:	
+	subcc	%g5, %g2, %g5			! hyper_flush_unconditional_combined
+	bne	1b
+	 sta	%g0, [%g5] ASI_M_FLUSH_CTX
+	retl
+	 sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE	! hyper_flush_whole_icache
+
+	/* We expand the window flush to get maximum performance. */
+hypersparc_flush_cache_mm:
+#ifndef CONFIG_SMP
+	ld	[%o0 + AOFF_mm_context], %g1
+	cmp	%g1, -1
+	be	hypersparc_flush_cache_mm_out
+#endif
+	WINDOW_FLUSH(%g4, %g5)
+
+	sethi	%hi(vac_line_size), %g1
+	ld	[%g1 + %lo(vac_line_size)], %o1
+	sethi	%hi(vac_cache_size), %g2
+	ld	[%g2 + %lo(vac_cache_size)], %o0
+	add	%o1, %o1, %g1
+	add	%o1, %g1, %g2
+	add	%o1, %g2, %g3
+	add	%o1, %g3, %g4
+	add	%o1, %g4, %g5
+	add	%o1, %g5, %o4
+	add	%o1, %o4, %o5
+
+	/* BLAMMO! */
+1:
+	subcc	%o0, %o5, %o0				! hyper_flush_cache_user
+	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_USER
+	sta	%g0, [%o0 + %o1] ASI_M_FLUSH_USER
+	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_USER
+	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_USER
+	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_USER
+	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_USER
+	sta	%g0, [%o0 + %g5] ASI_M_FLUSH_USER
+	bne	1b
+	 sta	%g0, [%o0 + %o4] ASI_M_FLUSH_USER
+hypersparc_flush_cache_mm_out:
+	retl
+	 nop
+
+	/* The things we do for performance... */
+hypersparc_flush_cache_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+#ifndef CONFIG_SMP
+	ld	[%o0 + AOFF_mm_context], %g1
+	cmp	%g1, -1
+	be	hypersparc_flush_cache_range_out
+#endif
+	WINDOW_FLUSH(%g4, %g5)
+
+	sethi	%hi(vac_line_size), %g1
+	ld	[%g1 + %lo(vac_line_size)], %o4
+	sethi	%hi(vac_cache_size), %g2
+	ld	[%g2 + %lo(vac_cache_size)], %o3
+
+	/* Here comes the fun part... */
+	add	%o2, (PAGE_SIZE - 1), %o2
+	andn	%o1, (PAGE_SIZE - 1), %o1
+	add	%o4, %o4, %o5
+	andn	%o2, (PAGE_SIZE - 1), %o2
+	add	%o4, %o5, %g1
+	sub	%o2, %o1, %g4
+	add	%o4, %g1, %g2
+	sll	%o3, 2, %g5
+	add	%o4, %g2, %g3
+	cmp	%g4, %g5
+	add	%o4, %g3, %g4
+	blu	0f
+	 add	%o4, %g4, %g5
+	add	%o4, %g5, %g7
+
+	/* Flush entire user space, believe it or not this is quicker
+	 * than page at a time flushings for range > (cache_size<<2).
+	 */
+1:
+	subcc	%o3, %g7, %o3
+	sta	%g0, [%o3 + %g0] ASI_M_FLUSH_USER
+	sta	%g0, [%o3 + %o4] ASI_M_FLUSH_USER
+	sta	%g0, [%o3 + %o5] ASI_M_FLUSH_USER
+	sta	%g0, [%o3 + %g1] ASI_M_FLUSH_USER
+	sta	%g0, [%o3 + %g2] ASI_M_FLUSH_USER
+	sta	%g0, [%o3 + %g3] ASI_M_FLUSH_USER
+	sta	%g0, [%o3 + %g4] ASI_M_FLUSH_USER
+	bne	1b
+	 sta	%g0, [%o3 + %g5] ASI_M_FLUSH_USER
+	retl
+	 nop
+
+	/* Below our threshold, flush one page at a time. */
+0:
+	ld	[%o0 + AOFF_mm_context], %o0
+	mov	SRMMU_CTX_REG, %g7
+	lda	[%g7] ASI_M_MMUREGS, %o3
+	sta	%o0, [%g7] ASI_M_MMUREGS
+	add	%o2, -PAGE_SIZE, %o0
+1:
+	or	%o0, 0x400, %g7
+	lda	[%g7] ASI_M_FLUSH_PROBE, %g7
+	orcc	%g7, 0, %g0
+	be,a	3f
+	 mov	%o0, %o2
+	add	%o4, %g5, %g7
+2:
+	sub	%o2, %g7, %o2
+	sta	%g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
+	andcc	%o2, 0xffc, %g0
+	sta	%g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
+	bne	2b
+	 sta	%g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
+3:
+	cmp	%o2, %o1
+	bne	1b
+	 add	%o2, -PAGE_SIZE, %o0
+	mov	SRMMU_FAULT_STATUS, %g5
+	lda	[%g5] ASI_M_MMUREGS, %g0
+	mov	SRMMU_CTX_REG, %g7
+	sta	%o3, [%g7] ASI_M_MMUREGS
+hypersparc_flush_cache_range_out:
+	retl
+	 nop
+
+	/* HyperSparc requires a valid mapping where we are about to flush
+	 * in order to check for a physical tag match during the flush.
+	 */
+	/* Verified, my ass... */
+hypersparc_flush_cache_page:
+	ld	[%o0 + VMA_VM_MM], %o0
+	ld	[%o0 + AOFF_mm_context], %g2
+#ifndef CONFIG_SMP
+	cmp	%g2, -1
+	be	hypersparc_flush_cache_page_out
+#endif
+	WINDOW_FLUSH(%g4, %g5)
+
+	sethi	%hi(vac_line_size), %g1
+	ld	[%g1 + %lo(vac_line_size)], %o4
+	mov	SRMMU_CTX_REG, %o3
+	andn	%o1, (PAGE_SIZE - 1), %o1
+	lda	[%o3] ASI_M_MMUREGS, %o2
+	sta	%g2, [%o3] ASI_M_MMUREGS
+	or	%o1, 0x400, %o5
+	lda	[%o5] ASI_M_FLUSH_PROBE, %g1
+	orcc	%g0, %g1, %g0
+	be	2f
+	 add	%o4, %o4, %o5
+	sub	%o1, -PAGE_SIZE, %o1
+	add	%o4, %o5, %g1
+	add	%o4, %g1, %g2
+	add	%o4, %g2, %g3
+	add	%o4, %g3, %g4
+	add	%o4, %g4, %g5
+	add	%o4, %g5, %g7
+
+	/* BLAMMO! */
+1:
+	sub	%o1, %g7, %o1
+	sta	%g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
+	andcc	%o1, 0xffc, %g0
+	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
+	bne	1b
+	 sta	%g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
+2:
+	mov	SRMMU_FAULT_STATUS, %g7
+	mov	SRMMU_CTX_REG, %g4
+	lda	[%g7] ASI_M_MMUREGS, %g0
+	sta	%o2, [%g4] ASI_M_MMUREGS
+hypersparc_flush_cache_page_out:
+	retl
+	 nop
+
+hypersparc_flush_sig_insns:
+	flush	%o1
+	retl
+	 flush	%o1 + 4
+
+	/* HyperSparc is copy-back. */
+hypersparc_flush_page_to_ram:
+	sethi	%hi(vac_line_size), %g1
+	ld	[%g1 + %lo(vac_line_size)], %o4
+	andn	%o0, (PAGE_SIZE - 1), %o0
+	add	%o4, %o4, %o5
+	or	%o0, 0x400, %g7
+	lda	[%g7] ASI_M_FLUSH_PROBE, %g5
+	add	%o4, %o5, %g1
+	orcc	%g5, 0, %g0
+	be	2f
+	 add	%o4, %g1, %g2
+	add	%o4, %g2, %g3
+	sub	%o0, -PAGE_SIZE, %o0
+	add	%o4, %g3, %g4
+	add	%o4, %g4, %g5
+	add	%o4, %g5, %g7
+
+	/* BLAMMO! */
+1:
+	sub	%o0, %g7, %o0
+	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
+	andcc	%o0, 0xffc, %g0
+	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
+	bne	1b
+	 sta	%g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
+2:
+	mov	SRMMU_FAULT_STATUS, %g1
+	retl
+	 lda	[%g1] ASI_M_MMUREGS, %g0
+
+	/* HyperSparc is IO cache coherent. */
+hypersparc_flush_page_for_dma:
+	retl
+	 nop
+
+	/* It was noted that at boot time a TLB flush all in a delay slot
+	 * can deliver an illegal instruction to the processor if the timing
+	 * is just right...
+	 */
+hypersparc_flush_tlb_all:
+	mov	0x400, %g1
+	sta	%g0, [%g1] ASI_M_FLUSH_PROBE
+	retl
+	 nop
+
+hypersparc_flush_tlb_mm:
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o1
+	lda	[%g1] ASI_M_MMUREGS, %g5
+#ifndef CONFIG_SMP
+	cmp	%o1, -1
+	be	hypersparc_flush_tlb_mm_out
+#endif
+	 mov	0x300, %g2
+	sta	%o1, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
+hypersparc_flush_tlb_mm_out:
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+
+hypersparc_flush_tlb_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o3
+	lda	[%g1] ASI_M_MMUREGS, %g5
+#ifndef CONFIG_SMP
+	cmp	%o3, -1
+	be	hypersparc_flush_tlb_range_out
+#endif
+	 sethi	%hi(~((1 << PGDIR_SHIFT) - 1)), %o4
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	and	%o1, %o4, %o1
+	add	%o1, 0x200, %o1
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+1:
+	sub	%o1, %o4, %o1
+	cmp	%o1, %o2
+	blu,a	1b
+	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+hypersparc_flush_tlb_range_out:
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+
+hypersparc_flush_tlb_page:
+	ld	[%o0 + VMA_VM_MM], %o0
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o3
+	andn	%o1, (PAGE_SIZE - 1), %o1
+#ifndef CONFIG_SMP
+	cmp	%o3, -1
+	be	hypersparc_flush_tlb_page_out
+#endif
+	 lda	[%g1] ASI_M_MMUREGS, %g5
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+hypersparc_flush_tlb_page_out:
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+
+	__INIT
+	
+	/* High speed page clear/copy. */
+hypersparc_bzero_1page:
+/* NOTE: This routine has to be shorter than 40insns --jj */
+	clr	%g1
+	mov	32, %g2
+	mov	64, %g3
+	mov	96, %g4
+	mov	128, %g5
+	mov	160, %g7
+	mov	192, %o2
+	mov	224, %o3
+	mov	16, %o1
+1:
+	stda	%g0, [%o0 + %g0] ASI_M_BFILL
+	stda	%g0, [%o0 + %g2] ASI_M_BFILL
+	stda	%g0, [%o0 + %g3] ASI_M_BFILL
+	stda	%g0, [%o0 + %g4] ASI_M_BFILL
+	stda	%g0, [%o0 + %g5] ASI_M_BFILL
+	stda	%g0, [%o0 + %g7] ASI_M_BFILL
+	stda	%g0, [%o0 + %o2] ASI_M_BFILL
+	stda	%g0, [%o0 + %o3] ASI_M_BFILL
+	subcc	%o1, 1, %o1
+	bne	1b
+	 add	%o0, 256, %o0
+
+	retl
+	 nop
+
+hypersparc_copy_1page:
+/* NOTE: This routine has to be shorter than 70insns --jj */
+	sub	%o1, %o0, %o2		! difference
+	mov	16, %g1
+1:
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	add	%o0, 32, %o0
+	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
+	subcc	%g1, 1, %g1
+	bne	1b
+	 add	%o0, 32, %o0
+
+	retl
+	 nop
+
+	.globl	hypersparc_setup_blockops
+hypersparc_setup_blockops:
+	sethi	%hi(bzero_1page), %o0
+	or	%o0, %lo(bzero_1page), %o0
+	sethi	%hi(hypersparc_bzero_1page), %o1
+	or	%o1, %lo(hypersparc_bzero_1page), %o1
+	sethi	%hi(hypersparc_copy_1page), %o2
+	or	%o2, %lo(hypersparc_copy_1page), %o2
+	ld	[%o1], %o4
+1:
+	add	%o1, 4, %o1
+	st	%o4, [%o0]
+	add	%o0, 4, %o0
+	cmp	%o1, %o2
+	bne	1b
+	 ld	[%o1], %o4
+	sethi	%hi(__copy_1page), %o0
+	or	%o0, %lo(__copy_1page), %o0
+	sethi	%hi(hypersparc_setup_blockops), %o2
+	or	%o2, %lo(hypersparc_setup_blockops), %o2
+	ld	[%o1], %o4
+1:
+	add	%o1, 4, %o1
+	st	%o4, [%o0]
+	add	%o0, 4, %o0
+	cmp	%o1, %o2
+	bne	1b
+	 ld	[%o1], %o4
+	sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE
+	retl
+	 nop
diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
new file mode 100644
index 000000000..e0e66f91c
--- /dev/null
+++ b/arch/sparc/mm/init_32.c
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  linux/arch/sparc/mm/init.c
+ *
+ *  Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
+ *  Copyright (C) 1995 Eddie C. Dost (ecd@skynet.be)
+ *  Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
+ *  Copyright (C) 2000 Anton Blanchard (anton@samba.org)
+ */
+
+#include <linux/module.h>
+#include <linux/signal.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/swap.h>
+#include <linux/initrd.h>
+#include <linux/init.h>
+#include <linux/highmem.h>
+#include <linux/memblock.h>
+#include <linux/pagemap.h>
+#include <linux/poison.h>
+#include <linux/gfp.h>
+
+#include <asm/sections.h>
+#include <asm/page.h>
+#include <asm/vaddrs.h>
+#include <asm/setup.h>
+#include <asm/tlb.h>
+#include <asm/prom.h>
+#include <asm/leon.h>
+
+#include "mm_32.h"
+
+static unsigned long *sparc_valid_addr_bitmap;
+
+unsigned long phys_base;
+EXPORT_SYMBOL(phys_base);
+
+unsigned long pfn_base;
+EXPORT_SYMBOL(pfn_base);
+
+struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1];
+
+/* Initial ramdisk setup */
+extern unsigned int sparc_ramdisk_image;
+extern unsigned int sparc_ramdisk_size;
+
+unsigned long highstart_pfn, highend_pfn;
+
+unsigned long last_valid_pfn;
+
+unsigned long calc_highpages(void)
+{
+	int i;
+	int nr = 0;
+
+	for (i = 0; sp_banks[i].num_bytes != 0; i++) {
+		unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT;
+		unsigned long end_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT;
+
+		if (end_pfn <= max_low_pfn)
+			continue;
+
+		if (start_pfn < max_low_pfn)
+			start_pfn = max_low_pfn;
+
+		nr += end_pfn - start_pfn;
+	}
+
+	return nr;
+}
+
+static unsigned long calc_max_low_pfn(void)
+{
+	int i;
+	unsigned long tmp = pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT);
+	unsigned long curr_pfn, last_pfn;
+
+	last_pfn = (sp_banks[0].base_addr + sp_banks[0].num_bytes) >> PAGE_SHIFT;
+	for (i = 1; sp_banks[i].num_bytes != 0; i++) {
+		curr_pfn = sp_banks[i].base_addr >> PAGE_SHIFT;
+
+		if (curr_pfn >= tmp) {
+			if (last_pfn < tmp)
+				tmp = last_pfn;
+			break;
+		}
+
+		last_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT;
+	}
+
+	return tmp;
+}
+
+static void __init find_ramdisk(unsigned long end_of_phys_memory)
+{
+#ifdef CONFIG_BLK_DEV_INITRD
+	unsigned long size;
+
+	/* Now have to check initial ramdisk, so that it won't pass
+	 * the end of memory
+	 */
+	if (sparc_ramdisk_image) {
+		if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
+			sparc_ramdisk_image -= KERNBASE;
+		initrd_start = sparc_ramdisk_image + phys_base;
+		initrd_end = initrd_start + sparc_ramdisk_size;
+		if (initrd_end > end_of_phys_memory) {
+			printk(KERN_CRIT "initrd extends beyond end of memory "
+			       "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
+			       initrd_end, end_of_phys_memory);
+			initrd_start = 0;
+		} else {
+			/* Reserve the initrd image area. */
+			size = initrd_end - initrd_start;
+			memblock_reserve(initrd_start, size);
+
+			initrd_start = (initrd_start - phys_base) + PAGE_OFFSET;
+			initrd_end = (initrd_end - phys_base) + PAGE_OFFSET;
+		}
+	}
+#endif
+}
+
+unsigned long __init bootmem_init(unsigned long *pages_avail)
+{
+	unsigned long start_pfn, bytes_avail, size;
+	unsigned long end_of_phys_memory = 0;
+	unsigned long high_pages = 0;
+	int i;
+
+	memblock_set_bottom_up(true);
+	memblock_allow_resize();
+
+	bytes_avail = 0UL;
+	for (i = 0; sp_banks[i].num_bytes != 0; i++) {
+		end_of_phys_memory = sp_banks[i].base_addr +
+			sp_banks[i].num_bytes;
+		bytes_avail += sp_banks[i].num_bytes;
+		if (cmdline_memory_size) {
+			if (bytes_avail > cmdline_memory_size) {
+				unsigned long slack = bytes_avail - cmdline_memory_size;
+
+				bytes_avail -= slack;
+				end_of_phys_memory -= slack;
+
+				sp_banks[i].num_bytes -= slack;
+				if (sp_banks[i].num_bytes == 0) {
+					sp_banks[i].base_addr = 0xdeadbeef;
+				} else {
+					memblock_add(sp_banks[i].base_addr,
+						     sp_banks[i].num_bytes);
+					sp_banks[i+1].num_bytes = 0;
+					sp_banks[i+1].base_addr = 0xdeadbeef;
+				}
+				break;
+			}
+		}
+		memblock_add(sp_banks[i].base_addr, sp_banks[i].num_bytes);
+	}
+
+	/* Start with page aligned address of last symbol in kernel
+	 * image.
+	 */
+	start_pfn  = (unsigned long)__pa(PAGE_ALIGN((unsigned long) &_end));
+
+	/* Now shift down to get the real physical page frame number. */
+	start_pfn >>= PAGE_SHIFT;
+
+	max_pfn = end_of_phys_memory >> PAGE_SHIFT;
+
+	max_low_pfn = max_pfn;
+	highstart_pfn = highend_pfn = max_pfn;
+
+	if (max_low_pfn > pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT)) {
+		highstart_pfn = pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT);
+		max_low_pfn = calc_max_low_pfn();
+		high_pages = calc_highpages();
+		printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
+		    high_pages >> (20 - PAGE_SHIFT));
+	}
+
+	find_ramdisk(end_of_phys_memory);
+
+	/* Reserve the kernel text/data/bss. */
+	size = (start_pfn << PAGE_SHIFT) - phys_base;
+	memblock_reserve(phys_base, size);
+	memblock_add(phys_base, size);
+
+	size = memblock_phys_mem_size() - memblock_reserved_size();
+	*pages_avail = (size >> PAGE_SHIFT) - high_pages;
+
+	/* Only allow low memory to be allocated via memblock allocation */
+	memblock_set_current_limit(max_low_pfn << PAGE_SHIFT);
+
+	return max_pfn;
+}
+
+/*
+ * paging_init() sets up the page tables: We call the MMU specific
+ * init routine based upon the Sun model type on the Sparc.
+ *
+ */
+void __init paging_init(void)
+{
+	srmmu_paging_init();
+	prom_build_devicetree();
+	of_fill_in_cpu_data();
+	device_scan();
+}
+
+static void __init taint_real_pages(void)
+{
+	int i;
+
+	for (i = 0; sp_banks[i].num_bytes; i++) {
+		unsigned long start, end;
+
+		start = sp_banks[i].base_addr;
+		end = start + sp_banks[i].num_bytes;
+
+		while (start < end) {
+			set_bit(start >> 20, sparc_valid_addr_bitmap);
+			start += PAGE_SIZE;
+		}
+	}
+}
+
+void __init arch_mm_preinit(void)
+{
+	int i;
+
+	if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) {
+		prom_printf("BUG: fixmap and pkmap areas overlap\n");
+		prom_printf("pkbase: 0x%lx pkend: 0x%lx fixstart 0x%lx\n",
+		       PKMAP_BASE,
+		       (unsigned long)PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
+		       FIXADDR_START);
+		prom_printf("Please mail sparclinux@vger.kernel.org.\n");
+		prom_halt();
+	}
+
+	i = last_valid_pfn >> ((20 - PAGE_SHIFT) + 5);
+	i += 1;
+	sparc_valid_addr_bitmap = (unsigned long *)
+		memblock_alloc(i << 2, SMP_CACHE_BYTES);
+
+	if (sparc_valid_addr_bitmap == NULL) {
+		prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
+		prom_halt();
+	}
+	memset(sparc_valid_addr_bitmap, 0, i << 2);
+
+	taint_real_pages();
+}
+
+void sparc_flush_page_to_ram(struct page *page)
+{
+	unsigned long vaddr = (unsigned long)page_address(page);
+
+	__flush_page_to_ram(vaddr);
+}
+EXPORT_SYMBOL(sparc_flush_page_to_ram);
+
+void sparc_flush_folio_to_ram(struct folio *folio)
+{
+	unsigned long vaddr = (unsigned long)folio_address(folio);
+	unsigned int i, nr = folio_nr_pages(folio);
+
+	for (i = 0; i < nr; i++)
+		__flush_page_to_ram(vaddr + i * PAGE_SIZE);
+}
+EXPORT_SYMBOL(sparc_flush_folio_to_ram);
+
+static const pgprot_t protection_map[16] = {
+	[VM_NONE]					= PAGE_NONE,
+	[VM_READ]					= PAGE_READONLY,
+	[VM_WRITE]					= PAGE_COPY,
+	[VM_WRITE | VM_READ]				= PAGE_COPY,
+	[VM_EXEC]					= PAGE_READONLY,
+	[VM_EXEC | VM_READ]				= PAGE_READONLY,
+	[VM_EXEC | VM_WRITE]				= PAGE_COPY,
+	[VM_EXEC | VM_WRITE | VM_READ]			= PAGE_COPY,
+	[VM_SHARED]					= PAGE_NONE,
+	[VM_SHARED | VM_READ]				= PAGE_READONLY,
+	[VM_SHARED | VM_WRITE]				= PAGE_SHARED,
+	[VM_SHARED | VM_WRITE | VM_READ]		= PAGE_SHARED,
+	[VM_SHARED | VM_EXEC]				= PAGE_READONLY,
+	[VM_SHARED | VM_EXEC | VM_READ]			= PAGE_READONLY,
+	[VM_SHARED | VM_EXEC | VM_WRITE]		= PAGE_SHARED,
+	[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ]	= PAGE_SHARED
+};
+DECLARE_VM_GET_PAGE_PROT
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
new file mode 100644
index 000000000..103db4683
--- /dev/null
+++ b/arch/sparc/mm/init_64.c
@@ -0,0 +1,3149 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  arch/sparc64/mm/init.c
+ *
+ *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
+ *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
+ */
+ 
+#include <linux/extable.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/string.h>
+#include <linux/init.h>
+#include <linux/memblock.h>
+#include <linux/mm.h>
+#include <linux/hugetlb.h>
+#include <linux/initrd.h>
+#include <linux/swap.h>
+#include <linux/pagemap.h>
+#include <linux/poison.h>
+#include <linux/fs.h>
+#include <linux/seq_file.h>
+#include <linux/kprobes.h>
+#include <linux/cache.h>
+#include <linux/sort.h>
+#include <linux/ioport.h>
+#include <linux/percpu.h>
+#include <linux/mmzone.h>
+#include <linux/gfp.h>
+
+#include <asm/head.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/oplib.h>
+#include <asm/iommu.h>
+#include <asm/io.h>
+#include <linux/uaccess.h>
+#include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
+#include <asm/dma.h>
+#include <asm/starfire.h>
+#include <asm/tlb.h>
+#include <asm/spitfire.h>
+#include <asm/sections.h>
+#include <asm/tsb.h>
+#include <asm/hypervisor.h>
+#include <asm/prom.h>
+#include <asm/mdesc.h>
+#include <asm/cpudata.h>
+#include <asm/setup.h>
+#include <asm/irq.h>
+
+#include "init_64.h"
+
+unsigned long kern_linear_pte_xor[4] __read_mostly;
+static unsigned long page_cache4v_flag;
+
+/* A bitmap, two bits for every 256MB of physical memory.  These two
+ * bits determine what page size we use for kernel linear
+ * translations.  They form an index into kern_linear_pte_xor[].  The
+ * value in the indexed slot is XOR'd with the TLB miss virtual
+ * address to form the resulting TTE.  The mapping is:
+ *
+ *	0	==>	4MB
+ *	1	==>	256MB
+ *	2	==>	2GB
+ *	3	==>	16GB
+ *
+ * All sun4v chips support 256MB pages.  Only SPARC-T4 and later
+ * support 2GB pages, and hopefully future cpus will support the 16GB
+ * pages as well.  For slots 2 and 3, we encode a 256MB TTE xor there
+ * if these larger page sizes are not supported by the cpu.
+ *
+ * It would be nice to determine this from the machine description
+ * 'cpu' properties, but we need to have this table setup before the
+ * MDESC is initialized.
+ */
+
+#ifndef CONFIG_DEBUG_PAGEALLOC
+/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
+ * Space is allocated for this right after the trap table in
+ * arch/sparc64/kernel/head.S
+ */
+extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
+#endif
+extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
+
+static unsigned long cpu_pgsz_mask;
+
+#define MAX_BANKS	1024
+
+static struct linux_prom64_registers pavail[MAX_BANKS];
+static int pavail_ents;
+
+u64 numa_latency[MAX_NUMNODES][MAX_NUMNODES];
+
+static int cmp_p64(const void *a, const void *b)
+{
+	const struct linux_prom64_registers *x = a, *y = b;
+
+	if (x->phys_addr > y->phys_addr)
+		return 1;
+	if (x->phys_addr < y->phys_addr)
+		return -1;
+	return 0;
+}
+
+static void __init read_obp_memory(const char *property,
+				   struct linux_prom64_registers *regs,
+				   int *num_ents)
+{
+	phandle node = prom_finddevice("/memory");
+	int prop_size = prom_getproplen(node, property);
+	int ents, ret, i;
+
+	ents = prop_size / sizeof(struct linux_prom64_registers);
+	if (ents > MAX_BANKS) {
+		prom_printf("The machine has more %s property entries than "
+			    "this kernel can support (%d).\n",
+			    property, MAX_BANKS);
+		prom_halt();
+	}
+
+	ret = prom_getproperty(node, property, (char *) regs, prop_size);
+	if (ret == -1) {
+		prom_printf("Couldn't get %s property from /memory.\n",
+				property);
+		prom_halt();
+	}
+
+	/* Sanitize what we got from the firmware, by page aligning
+	 * everything.
+	 */
+	for (i = 0; i < ents; i++) {
+		unsigned long base, size;
+
+		base = regs[i].phys_addr;
+		size = regs[i].reg_size;
+
+		size &= PAGE_MASK;
+		if (base & ~PAGE_MASK) {
+			unsigned long new_base = PAGE_ALIGN(base);
+
+			size -= new_base - base;
+			if ((long) size < 0L)
+				size = 0UL;
+			base = new_base;
+		}
+		if (size == 0UL) {
+			/* If it is empty, simply get rid of it.
+			 * This simplifies the logic of the other
+			 * functions that process these arrays.
+			 */
+			memmove(&regs[i], &regs[i + 1],
+				(ents - i - 1) * sizeof(regs[0]));
+			i--;
+			ents--;
+			continue;
+		}
+		regs[i].phys_addr = base;
+		regs[i].reg_size = size;
+	}
+
+	*num_ents = ents;
+
+	sort(regs, ents, sizeof(struct linux_prom64_registers),
+	     cmp_p64, NULL);
+}
+
+/* Kernel physical address base and size in bytes.  */
+unsigned long kern_base __read_mostly;
+unsigned long kern_size __read_mostly;
+
+/* Initial ramdisk setup */
+extern unsigned long sparc_ramdisk_image64;
+extern unsigned int sparc_ramdisk_image;
+extern unsigned int sparc_ramdisk_size;
+
+unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
+
+unsigned long sparc64_kern_pri_context __read_mostly;
+unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
+unsigned long sparc64_kern_sec_context __read_mostly;
+
+int num_kernel_image_mappings;
+
+#ifdef CONFIG_DEBUG_DCFLUSH
+atomic_t dcpage_flushes = ATOMIC_INIT(0);
+#ifdef CONFIG_SMP
+atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
+#endif
+#endif
+
+inline void flush_dcache_folio_impl(struct folio *folio)
+{
+	unsigned int i, nr = folio_nr_pages(folio);
+
+	BUG_ON(tlb_type == hypervisor);
+#ifdef CONFIG_DEBUG_DCFLUSH
+	atomic_inc(&dcpage_flushes);
+#endif
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+	for (i = 0; i < nr; i++)
+		__flush_dcache_page(folio_address(folio) + i * PAGE_SIZE,
+				    ((tlb_type == spitfire) &&
+				     folio_flush_mapping(folio) != NULL));
+#else
+	if (folio_flush_mapping(folio) != NULL &&
+	    tlb_type == spitfire) {
+		for (i = 0; i < nr; i++)
+			__flush_icache_page((pfn + i) * PAGE_SIZE);
+	}
+#endif
+}
+
+#define PG_dcache_dirty		PG_arch_1
+#define PG_dcache_cpu_shift	32UL
+#define PG_dcache_cpu_mask	\
+	((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
+
+#define dcache_dirty_cpu(folio) \
+	(((folio)->flags.f >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
+
+static inline void set_dcache_dirty(struct folio *folio, int this_cpu)
+{
+	unsigned long mask = this_cpu;
+	unsigned long non_cpu_bits;
+
+	non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
+	mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
+
+	__asm__ __volatile__("1:\n\t"
+			     "ldx	[%2], %%g7\n\t"
+			     "and	%%g7, %1, %%g1\n\t"
+			     "or	%%g1, %0, %%g1\n\t"
+			     "casx	[%2], %%g7, %%g1\n\t"
+			     "cmp	%%g7, %%g1\n\t"
+			     "bne,pn	%%xcc, 1b\n\t"
+			     " nop"
+			     : /* no outputs */
+			     : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags.f)
+			     : "g1", "g7");
+}
+
+static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu)
+{
+	unsigned long mask = (1UL << PG_dcache_dirty);
+
+	__asm__ __volatile__("! test_and_clear_dcache_dirty\n"
+			     "1:\n\t"
+			     "ldx	[%2], %%g7\n\t"
+			     "srlx	%%g7, %4, %%g1\n\t"
+			     "and	%%g1, %3, %%g1\n\t"
+			     "cmp	%%g1, %0\n\t"
+			     "bne,pn	%%icc, 2f\n\t"
+			     " andn	%%g7, %1, %%g1\n\t"
+			     "casx	[%2], %%g7, %%g1\n\t"
+			     "cmp	%%g7, %%g1\n\t"
+			     "bne,pn	%%xcc, 1b\n\t"
+			     " nop\n"
+			     "2:"
+			     : /* no outputs */
+			     : "r" (cpu), "r" (mask), "r" (&folio->flags.f),
+			       "i" (PG_dcache_cpu_mask),
+			       "i" (PG_dcache_cpu_shift)
+			     : "g1", "g7");
+}
+
+static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
+{
+	unsigned long tsb_addr = (unsigned long) ent;
+
+	if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+		tsb_addr = __pa(tsb_addr);
+
+	__tsb_insert(tsb_addr, tag, pte);
+}
+
+unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
+
+static void flush_dcache(unsigned long pfn)
+{
+	struct page *page;
+
+	page = pfn_to_page(pfn);
+	if (page) {
+		struct folio *folio = page_folio(page);
+		unsigned long pg_flags;
+
+		pg_flags = folio->flags.f;
+		if (pg_flags & (1UL << PG_dcache_dirty)) {
+			int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
+				   PG_dcache_cpu_mask);
+			int this_cpu = get_cpu();
+
+			/* This is just to optimize away some function calls
+			 * in the SMP case.
+			 */
+			if (cpu == this_cpu)
+				flush_dcache_folio_impl(folio);
+			else
+				smp_flush_dcache_folio_impl(folio, cpu);
+
+			clear_dcache_dirty_cpu(folio, cpu);
+
+			put_cpu();
+		}
+	}
+}
+
+/* mm->context.lock must be held */
+static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,
+				    unsigned long tsb_hash_shift, unsigned long address,
+				    unsigned long tte)
+{
+	struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
+	unsigned long tag;
+
+	if (unlikely(!tsb))
+		return;
+
+	tsb += ((address >> tsb_hash_shift) &
+		(mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
+	tag = (address >> 22UL);
+	tsb_insert(tsb, tag, tte);
+}
+
+#ifdef CONFIG_HUGETLB_PAGE
+static int __init hugetlbpage_init(void)
+{
+	hugetlb_add_hstate(HPAGE_64K_SHIFT - PAGE_SHIFT);
+	hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT);
+	hugetlb_add_hstate(HPAGE_256MB_SHIFT - PAGE_SHIFT);
+	hugetlb_add_hstate(HPAGE_2GB_SHIFT - PAGE_SHIFT);
+
+	return 0;
+}
+
+arch_initcall(hugetlbpage_init);
+
+static void __init pud_huge_patch(void)
+{
+	struct pud_huge_patch_entry *p;
+	unsigned long addr;
+
+	p = &__pud_huge_patch;
+	addr = p->addr;
+	*(unsigned int *)addr = p->insn;
+
+	__asm__ __volatile__("flush %0" : : "r" (addr));
+}
+
+bool __init arch_hugetlb_valid_size(unsigned long size)
+{
+	unsigned int hugepage_shift = ilog2(size);
+	unsigned int hv_pgsz_mask;
+
+	switch (hugepage_shift) {
+	case HPAGE_16GB_SHIFT:
+		hv_pgsz_mask = HV_PGSZ_MASK_16GB;
+		pud_huge_patch();
+		break;
+	case HPAGE_2GB_SHIFT:
+		hv_pgsz_mask = HV_PGSZ_MASK_2GB;
+		break;
+	case HPAGE_256MB_SHIFT:
+		hv_pgsz_mask = HV_PGSZ_MASK_256MB;
+		break;
+	case HPAGE_SHIFT:
+		hv_pgsz_mask = HV_PGSZ_MASK_4MB;
+		break;
+	case HPAGE_64K_SHIFT:
+		hv_pgsz_mask = HV_PGSZ_MASK_64K;
+		break;
+	default:
+		hv_pgsz_mask = 0;
+	}
+
+	if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U)
+		return false;
+
+	return true;
+}
+#endif	/* CONFIG_HUGETLB_PAGE */
+
+void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma,
+		unsigned long address, pte_t *ptep, unsigned int nr)
+{
+	struct mm_struct *mm;
+	unsigned long flags;
+	bool is_huge_tsb;
+	pte_t pte = *ptep;
+	unsigned int i;
+
+	if (tlb_type != hypervisor) {
+		unsigned long pfn = pte_pfn(pte);
+
+		if (pfn_valid(pfn))
+			flush_dcache(pfn);
+	}
+
+	mm = vma->vm_mm;
+
+	/* Don't insert a non-valid PTE into the TSB, we'll deadlock.  */
+	if (!pte_accessible(mm, pte))
+		return;
+
+	spin_lock_irqsave(&mm->context.lock, flags);
+
+	is_huge_tsb = false;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	if (mm->context.hugetlb_pte_count || mm->context.thp_pte_count) {
+		unsigned long hugepage_size = PAGE_SIZE;
+
+		if (is_vm_hugetlb_page(vma))
+			hugepage_size = huge_page_size(hstate_vma(vma));
+
+		if (hugepage_size >= PUD_SIZE) {
+			unsigned long mask = 0x1ffc00000UL;
+
+			/* Transfer bits [32:22] from address to resolve
+			 * at 4M granularity.
+			 */
+			pte_val(pte) &= ~mask;
+			pte_val(pte) |= (address & mask);
+		} else if (hugepage_size >= PMD_SIZE) {
+			/* We are fabricating 8MB pages using 4MB
+			 * real hw pages.
+			 */
+			pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT));
+		}
+
+		if (hugepage_size >= PMD_SIZE) {
+			__update_mmu_tsb_insert(mm, MM_TSB_HUGE,
+				REAL_HPAGE_SHIFT, address, pte_val(pte));
+			is_huge_tsb = true;
+		}
+	}
+#endif
+	if (!is_huge_tsb) {
+		for (i = 0; i < nr; i++) {
+			__update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
+						address, pte_val(pte));
+			address += PAGE_SIZE;
+			pte_val(pte) += PAGE_SIZE;
+		}
+	}
+
+	spin_unlock_irqrestore(&mm->context.lock, flags);
+}
+
+void flush_dcache_folio(struct folio *folio)
+{
+	unsigned long pfn = folio_pfn(folio);
+	struct address_space *mapping;
+	int this_cpu;
+
+	if (tlb_type == hypervisor)
+		return;
+
+	/* Do not bother with the expensive D-cache flush if it
+	 * is merely the zero page.  The 'bigcore' testcase in GDB
+	 * causes this case to run millions of times.
+	 */
+	if (is_zero_pfn(pfn))
+		return;
+
+	this_cpu = get_cpu();
+
+	mapping = folio_flush_mapping(folio);
+	if (mapping && !mapping_mapped(mapping)) {
+		bool dirty = test_bit(PG_dcache_dirty, &folio->flags.f);
+		if (dirty) {
+			int dirty_cpu = dcache_dirty_cpu(folio);
+
+			if (dirty_cpu == this_cpu)
+				goto out;
+			smp_flush_dcache_folio_impl(folio, dirty_cpu);
+		}
+		set_dcache_dirty(folio, this_cpu);
+	} else {
+		/* We could delay the flush for the !folio_mapping
+		 * case too.  But that case is for exec env/arg
+		 * pages and those are %99 certainly going to get
+		 * faulted into the tlb (and thus flushed) anyways.
+		 */
+		flush_dcache_folio_impl(folio);
+	}
+
+out:
+	put_cpu();
+}
+EXPORT_SYMBOL(flush_dcache_folio);
+
+void __kprobes flush_icache_range(unsigned long start, unsigned long end)
+{
+	/* Cheetah and Hypervisor platform cpus have coherent I-cache. */
+	if (tlb_type == spitfire) {
+		unsigned long kaddr;
+
+		/* This code only runs on Spitfire cpus so this is
+		 * why we can assume _PAGE_PADDR_4U.
+		 */
+		for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
+			unsigned long paddr, mask = _PAGE_PADDR_4U;
+
+			if (kaddr >= PAGE_OFFSET)
+				paddr = kaddr & mask;
+			else {
+				pte_t *ptep = virt_to_kpte(kaddr);
+
+				paddr = pte_val(*ptep) & mask;
+			}
+			__flush_icache_page(paddr);
+		}
+	}
+}
+EXPORT_SYMBOL(flush_icache_range);
+
+void mmu_info(struct seq_file *m)
+{
+	static const char *pgsz_strings[] = {
+		"8K", "64K", "512K", "4MB", "32MB",
+		"256MB", "2GB", "16GB",
+	};
+	int i, printed;
+
+	if (tlb_type == cheetah)
+		seq_printf(m, "MMU Type\t: Cheetah\n");
+	else if (tlb_type == cheetah_plus)
+		seq_printf(m, "MMU Type\t: Cheetah+\n");
+	else if (tlb_type == spitfire)
+		seq_printf(m, "MMU Type\t: Spitfire\n");
+	else if (tlb_type == hypervisor)
+		seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
+	else
+		seq_printf(m, "MMU Type\t: ???\n");
+
+	seq_printf(m, "MMU PGSZs\t: ");
+	printed = 0;
+	for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {
+		if (cpu_pgsz_mask & (1UL << i)) {
+			seq_printf(m, "%s%s",
+				   printed ? "," : "", pgsz_strings[i]);
+			printed++;
+		}
+	}
+	seq_putc(m, '\n');
+
+#ifdef CONFIG_DEBUG_DCFLUSH
+	seq_printf(m, "DCPageFlushes\t: %d\n",
+		   atomic_read(&dcpage_flushes));
+#ifdef CONFIG_SMP
+	seq_printf(m, "DCPageFlushesXC\t: %d\n",
+		   atomic_read(&dcpage_flushes_xcall));
+#endif /* CONFIG_SMP */
+#endif /* CONFIG_DEBUG_DCFLUSH */
+}
+
+struct linux_prom_translation prom_trans[512] __read_mostly;
+unsigned int prom_trans_ents __read_mostly;
+
+unsigned long kern_locked_tte_data;
+
+/* The obp translations are saved based on 8k pagesize, since obp can
+ * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
+ * HI_OBP_ADDRESS range are handled in ktlb.S.
+ */
+static inline int in_obp_range(unsigned long vaddr)
+{
+	return (vaddr >= LOW_OBP_ADDRESS &&
+		vaddr < HI_OBP_ADDRESS);
+}
+
+static int cmp_ptrans(const void *a, const void *b)
+{
+	const struct linux_prom_translation *x = a, *y = b;
+
+	if (x->virt > y->virt)
+		return 1;
+	if (x->virt < y->virt)
+		return -1;
+	return 0;
+}
+
+/* Read OBP translations property into 'prom_trans[]'.  */
+static void __init read_obp_translations(void)
+{
+	int n, node, ents, first, last, i;
+
+	node = prom_finddevice("/virtual-memory");
+	n = prom_getproplen(node, "translations");
+	if (unlikely(n == 0 || n == -1)) {
+		prom_printf("prom_mappings: Couldn't get size.\n");
+		prom_halt();
+	}
+	if (unlikely(n > sizeof(prom_trans))) {
+		prom_printf("prom_mappings: Size %d is too big.\n", n);
+		prom_halt();
+	}
+
+	if ((n = prom_getproperty(node, "translations",
+				  (char *)&prom_trans[0],
+				  sizeof(prom_trans))) == -1) {
+		prom_printf("prom_mappings: Couldn't get property.\n");
+		prom_halt();
+	}
+
+	n = n / sizeof(struct linux_prom_translation);
+
+	ents = n;
+
+	sort(prom_trans, ents, sizeof(struct linux_prom_translation),
+	     cmp_ptrans, NULL);
+
+	/* Now kick out all the non-OBP entries.  */
+	for (i = 0; i < ents; i++) {
+		if (in_obp_range(prom_trans[i].virt))
+			break;
+	}
+	first = i;
+	for (; i < ents; i++) {
+		if (!in_obp_range(prom_trans[i].virt))
+			break;
+	}
+	last = i;
+
+	for (i = 0; i < (last - first); i++) {
+		struct linux_prom_translation *src = &prom_trans[i + first];
+		struct linux_prom_translation *dest = &prom_trans[i];
+
+		*dest = *src;
+	}
+	for (; i < ents; i++) {
+		struct linux_prom_translation *dest = &prom_trans[i];
+		dest->virt = dest->size = dest->data = 0x0UL;
+	}
+
+	prom_trans_ents = last - first;
+
+	if (tlb_type == spitfire) {
+		/* Clear diag TTE bits. */
+		for (i = 0; i < prom_trans_ents; i++)
+			prom_trans[i].data &= ~0x0003fe0000000000UL;
+	}
+
+	/* Force execute bit on.  */
+	for (i = 0; i < prom_trans_ents; i++)
+		prom_trans[i].data |= (tlb_type == hypervisor ?
+				       _PAGE_EXEC_4V : _PAGE_EXEC_4U);
+}
+
+static void __init hypervisor_tlb_lock(unsigned long vaddr,
+				       unsigned long pte,
+				       unsigned long mmu)
+{
+	unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
+
+	if (ret != 0) {
+		prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
+			    "errors with %lx\n", vaddr, 0, pte, mmu, ret);
+		prom_halt();
+	}
+}
+
+static unsigned long kern_large_tte(unsigned long paddr);
+
+static void __init remap_kernel(void)
+{
+	unsigned long phys_page, tte_vaddr, tte_data;
+	int i, tlb_ent = sparc64_highest_locked_tlbent();
+
+	tte_vaddr = (unsigned long) KERNBASE;
+	phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
+	tte_data = kern_large_tte(phys_page);
+
+	kern_locked_tte_data = tte_data;
+
+	/* Now lock us into the TLBs via Hypervisor or OBP. */
+	if (tlb_type == hypervisor) {
+		for (i = 0; i < num_kernel_image_mappings; i++) {
+			hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
+			hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
+			tte_vaddr += 0x400000;
+			tte_data += 0x400000;
+		}
+	} else {
+		for (i = 0; i < num_kernel_image_mappings; i++) {
+			prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);
+			prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);
+			tte_vaddr += 0x400000;
+			tte_data += 0x400000;
+		}
+		sparc64_highest_unlocked_tlb_ent = tlb_ent - i;
+	}
+	if (tlb_type == cheetah_plus) {
+		sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
+					    CTX_CHEETAH_PLUS_NUC);
+		sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
+		sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
+	}
+}
+
+
+static void __init inherit_prom_mappings(void)
+{
+	/* Now fixup OBP's idea about where we really are mapped. */
+	printk("Remapping the kernel... ");
+	remap_kernel();
+	printk("done.\n");
+}
+
+void prom_world(int enter)
+{
+	/*
+	 * No need to change the address space any more, just flush
+	 * the register windows
+	 */
+	__asm__ __volatile__("flushw");
+}
+
+void __flush_dcache_range(unsigned long start, unsigned long end)
+{
+	unsigned long va;
+
+	if (tlb_type == spitfire) {
+		int n = 0;
+
+		for (va = start; va < end; va += 32) {
+			spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
+			if (++n >= 512)
+				break;
+		}
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
+		start = __pa(start);
+		end = __pa(end);
+		for (va = start; va < end; va += 32)
+			__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+					     "membar #Sync"
+					     : /* no outputs */
+					     : "r" (va),
+					       "i" (ASI_DCACHE_INVALIDATE));
+	}
+}
+EXPORT_SYMBOL(__flush_dcache_range);
+
+/* get_new_mmu_context() uses "cache + 1".  */
+DEFINE_SPINLOCK(ctx_alloc_lock);
+unsigned long tlb_context_cache = CTX_FIRST_VERSION;
+#define MAX_CTX_NR	(1UL << CTX_NR_BITS)
+#define CTX_BMAP_SLOTS	BITS_TO_LONGS(MAX_CTX_NR)
+DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);
+DEFINE_PER_CPU(struct mm_struct *, per_cpu_secondary_mm) = {0};
+
+static void mmu_context_wrap(void)
+{
+	unsigned long old_ver = tlb_context_cache & CTX_VERSION_MASK;
+	unsigned long new_ver, new_ctx, old_ctx;
+	struct mm_struct *mm;
+	int cpu;
+
+	bitmap_zero(mmu_context_bmap, 1 << CTX_NR_BITS);
+
+	/* Reserve kernel context */
+	set_bit(0, mmu_context_bmap);
+
+	new_ver = (tlb_context_cache & CTX_VERSION_MASK) + CTX_FIRST_VERSION;
+	if (unlikely(new_ver == 0))
+		new_ver = CTX_FIRST_VERSION;
+	tlb_context_cache = new_ver;
+
+	/*
+	 * Make sure that any new mm that are added into per_cpu_secondary_mm,
+	 * are going to go through get_new_mmu_context() path.
+	 */
+	mb();
+
+	/*
+	 * Updated versions to current on those CPUs that had valid secondary
+	 * contexts
+	 */
+	for_each_online_cpu(cpu) {
+		/*
+		 * If a new mm is stored after we took this mm from the array,
+		 * it will go into get_new_mmu_context() path, because we
+		 * already bumped the version in tlb_context_cache.
+		 */
+		mm = per_cpu(per_cpu_secondary_mm, cpu);
+
+		if (unlikely(!mm || mm == &init_mm))
+			continue;
+
+		old_ctx = mm->context.sparc64_ctx_val;
+		if (likely((old_ctx & CTX_VERSION_MASK) == old_ver)) {
+			new_ctx = (old_ctx & ~CTX_VERSION_MASK) | new_ver;
+			set_bit(new_ctx & CTX_NR_MASK, mmu_context_bmap);
+			mm->context.sparc64_ctx_val = new_ctx;
+		}
+	}
+}
+
+/* Caller does TLB context flushing on local CPU if necessary.
+ * The caller also ensures that CTX_VALID(mm->context) is false.
+ *
+ * We must be careful about boundary cases so that we never
+ * let the user have CTX 0 (nucleus) or we ever use a CTX
+ * version of zero (and thus NO_CONTEXT would not be caught
+ * by version mis-match tests in mmu_context.h).
+ *
+ * Always invoked with interrupts disabled.
+ */
+void get_new_mmu_context(struct mm_struct *mm)
+{
+	unsigned long ctx, new_ctx;
+	unsigned long orig_pgsz_bits;
+
+	spin_lock(&ctx_alloc_lock);
+retry:
+	/* wrap might have happened, test again if our context became valid */
+	if (unlikely(CTX_VALID(mm->context)))
+		goto out;
+	orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
+	ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
+	new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
+	if (new_ctx >= (1 << CTX_NR_BITS)) {
+		new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
+		if (new_ctx >= ctx) {
+			mmu_context_wrap();
+			goto retry;
+		}
+	}
+	if (mm->context.sparc64_ctx_val)
+		cpumask_clear(mm_cpumask(mm));
+	mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
+	new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
+	tlb_context_cache = new_ctx;
+	mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
+out:
+	spin_unlock(&ctx_alloc_lock);
+}
+
+static int numa_enabled = 1;
+static int numa_debug;
+
+static int __init early_numa(char *p)
+{
+	if (!p)
+		return 0;
+
+	if (strstr(p, "off"))
+		numa_enabled = 0;
+
+	if (strstr(p, "debug"))
+		numa_debug = 1;
+
+	return 0;
+}
+early_param("numa", early_numa);
+
+#define numadbg(f, a...) \
+do {	if (numa_debug) \
+		printk(KERN_INFO f, ## a); \
+} while (0)
+
+static void __init find_ramdisk(unsigned long phys_base)
+{
+#ifdef CONFIG_BLK_DEV_INITRD
+	if (sparc_ramdisk_image || sparc_ramdisk_image64) {
+		unsigned long ramdisk_image;
+
+		/* Older versions of the bootloader only supported a
+		 * 32-bit physical address for the ramdisk image
+		 * location, stored at sparc_ramdisk_image.  Newer
+		 * SILO versions set sparc_ramdisk_image to zero and
+		 * provide a full 64-bit physical address at
+		 * sparc_ramdisk_image64.
+		 */
+		ramdisk_image = sparc_ramdisk_image;
+		if (!ramdisk_image)
+			ramdisk_image = sparc_ramdisk_image64;
+
+		/* Another bootloader quirk.  The bootloader normalizes
+		 * the physical address to KERNBASE, so we have to
+		 * factor that back out and add in the lowest valid
+		 * physical page address to get the true physical address.
+		 */
+		ramdisk_image -= KERNBASE;
+		ramdisk_image += phys_base;
+
+		numadbg("Found ramdisk at physical address 0x%lx, size %u\n",
+			ramdisk_image, sparc_ramdisk_size);
+
+		initrd_start = ramdisk_image;
+		initrd_end = ramdisk_image + sparc_ramdisk_size;
+
+		memblock_reserve(initrd_start, sparc_ramdisk_size);
+
+		initrd_start += PAGE_OFFSET;
+		initrd_end += PAGE_OFFSET;
+	}
+#endif
+}
+
+struct node_mem_mask {
+	unsigned long mask;
+	unsigned long match;
+};
+static struct node_mem_mask node_masks[MAX_NUMNODES];
+static int num_node_masks;
+
+#ifdef CONFIG_NUMA
+
+struct mdesc_mlgroup {
+	u64	node;
+	u64	latency;
+	u64	match;
+	u64	mask;
+};
+
+static struct mdesc_mlgroup *mlgroups;
+static int num_mlgroups;
+
+int numa_cpu_lookup_table[NR_CPUS];
+cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
+
+struct mdesc_mblock {
+	u64	base;
+	u64	size;
+	u64	offset; /* RA-to-PA */
+};
+static struct mdesc_mblock *mblocks;
+static int num_mblocks;
+
+static struct mdesc_mblock * __init addr_to_mblock(unsigned long addr)
+{
+	struct mdesc_mblock *m = NULL;
+	int i;
+
+	for (i = 0; i < num_mblocks; i++) {
+		m = &mblocks[i];
+
+		if (addr >= m->base &&
+		    addr < (m->base + m->size)) {
+			break;
+		}
+	}
+
+	return m;
+}
+
+static u64 __init memblock_nid_range_sun4u(u64 start, u64 end, int *nid)
+{
+	int prev_nid, new_nid;
+
+	prev_nid = NUMA_NO_NODE;
+	for ( ; start < end; start += PAGE_SIZE) {
+		for (new_nid = 0; new_nid < num_node_masks; new_nid++) {
+			struct node_mem_mask *p = &node_masks[new_nid];
+
+			if ((start & p->mask) == p->match) {
+				if (prev_nid == NUMA_NO_NODE)
+					prev_nid = new_nid;
+				break;
+			}
+		}
+
+		if (new_nid == num_node_masks) {
+			prev_nid = 0;
+			WARN_ONCE(1, "addr[%Lx] doesn't match a NUMA node rule. Some memory will be owned by node 0.",
+				  start);
+			break;
+		}
+
+		if (prev_nid != new_nid)
+			break;
+	}
+	*nid = prev_nid;
+
+	return start > end ? end : start;
+}
+
+static u64 __init memblock_nid_range(u64 start, u64 end, int *nid)
+{
+	u64 ret_end, pa_start, m_mask, m_match, m_end;
+	struct mdesc_mblock *mblock;
+	int _nid, i;
+
+	if (tlb_type != hypervisor)
+		return memblock_nid_range_sun4u(start, end, nid);
+
+	mblock = addr_to_mblock(start);
+	if (!mblock) {
+		WARN_ONCE(1, "memblock_nid_range: Can't find mblock addr[%Lx]",
+			  start);
+
+		_nid = 0;
+		ret_end = end;
+		goto done;
+	}
+
+	pa_start = start + mblock->offset;
+	m_match = 0;
+	m_mask = 0;
+
+	for (_nid = 0; _nid < num_node_masks; _nid++) {
+		struct node_mem_mask *const m = &node_masks[_nid];
+
+		if ((pa_start & m->mask) == m->match) {
+			m_match = m->match;
+			m_mask = m->mask;
+			break;
+		}
+	}
+
+	if (num_node_masks == _nid) {
+		/* We could not find NUMA group, so default to 0, but lets
+		 * search for latency group, so we could calculate the correct
+		 * end address that we return
+		 */
+		_nid = 0;
+
+		for (i = 0; i < num_mlgroups; i++) {
+			struct mdesc_mlgroup *const m = &mlgroups[i];
+
+			if ((pa_start & m->mask) == m->match) {
+				m_match = m->match;
+				m_mask = m->mask;
+				break;
+			}
+		}
+
+		if (i == num_mlgroups) {
+			WARN_ONCE(1, "memblock_nid_range: Can't find latency group addr[%Lx]",
+				  start);
+
+			ret_end = end;
+			goto done;
+		}
+	}
+
+	/*
+	 * Each latency group has match and mask, and each memory block has an
+	 * offset.  An address belongs to a latency group if its address matches
+	 * the following formula: ((addr + offset) & mask) == match
+	 * It is, however, slow to check every single page if it matches a
+	 * particular latency group. As optimization we calculate end value by
+	 * using bit arithmetics.
+	 */
+	m_end = m_match + (1ul << __ffs(m_mask)) - mblock->offset;
+	m_end += pa_start & ~((1ul << fls64(m_mask)) - 1);
+	ret_end = m_end > end ? end : m_end;
+
+done:
+	*nid = _nid;
+	return ret_end;
+}
+#endif
+
+/* This must be invoked after performing all of the necessary
+ * memblock_set_node() calls for 'nid'.  We need to be able to get
+ * correct data from get_pfn_range_for_nid().
+ */
+static void __init allocate_node_data(int nid)
+{
+	struct pglist_data *p;
+	unsigned long start_pfn, end_pfn;
+
+#ifdef CONFIG_NUMA
+	alloc_node_data(nid);
+
+	NODE_DATA(nid)->node_id = nid;
+#endif
+
+	p = NODE_DATA(nid);
+
+	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
+	p->node_start_pfn = start_pfn;
+	p->node_spanned_pages = end_pfn - start_pfn;
+}
+
+static void init_node_masks_nonnuma(void)
+{
+#ifdef CONFIG_NUMA
+	int i;
+#endif
+
+	numadbg("Initializing tables for non-numa.\n");
+
+	node_masks[0].mask = 0;
+	node_masks[0].match = 0;
+	num_node_masks = 1;
+
+#ifdef CONFIG_NUMA
+	for (i = 0; i < NR_CPUS; i++)
+		numa_cpu_lookup_table[i] = 0;
+
+	cpumask_setall(&numa_cpumask_lookup_table[0]);
+#endif
+}
+
+#ifdef CONFIG_NUMA
+
+EXPORT_SYMBOL(numa_cpu_lookup_table);
+EXPORT_SYMBOL(numa_cpumask_lookup_table);
+
+static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,
+				   u32 cfg_handle)
+{
+	u64 arc;
+
+	mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {
+		u64 target = mdesc_arc_target(md, arc);
+		const u64 *val;
+
+		val = mdesc_get_property(md, target,
+					 "cfg-handle", NULL);
+		if (val && *val == cfg_handle)
+			return 0;
+	}
+	return -ENODEV;
+}
+
+static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,
+				    u32 cfg_handle)
+{
+	u64 arc, candidate, best_latency = ~(u64)0;
+
+	candidate = MDESC_NODE_NULL;
+	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
+		u64 target = mdesc_arc_target(md, arc);
+		const char *name = mdesc_node_name(md, target);
+		const u64 *val;
+
+		if (strcmp(name, "pio-latency-group"))
+			continue;
+
+		val = mdesc_get_property(md, target, "latency", NULL);
+		if (!val)
+			continue;
+
+		if (*val < best_latency) {
+			candidate = target;
+			best_latency = *val;
+		}
+	}
+
+	if (candidate == MDESC_NODE_NULL)
+		return -ENODEV;
+
+	return scan_pio_for_cfg_handle(md, candidate, cfg_handle);
+}
+
+int of_node_to_nid(struct device_node *dp)
+{
+	const struct linux_prom64_registers *regs;
+	struct mdesc_handle *md;
+	u32 cfg_handle;
+	int count, nid;
+	u64 grp;
+
+	/* This is the right thing to do on currently supported
+	 * SUN4U NUMA platforms as well, as the PCI controller does
+	 * not sit behind any particular memory controller.
+	 */
+	if (!mlgroups)
+		return -1;
+
+	regs = of_get_property(dp, "reg", NULL);
+	if (!regs)
+		return -1;
+
+	cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;
+
+	md = mdesc_grab();
+
+	count = 0;
+	nid = NUMA_NO_NODE;
+	mdesc_for_each_node_by_name(md, grp, "group") {
+		if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {
+			nid = count;
+			break;
+		}
+		count++;
+	}
+
+	mdesc_release(md);
+
+	return nid;
+}
+
+static void __init add_node_ranges(void)
+{
+	phys_addr_t start, end;
+	unsigned long prev_max;
+	u64 i;
+
+memblock_resized:
+	prev_max = memblock.memory.max;
+
+	for_each_mem_range(i, &start, &end) {
+		while (start < end) {
+			unsigned long this_end;
+			int nid;
+
+			this_end = memblock_nid_range(start, end, &nid);
+
+			numadbg("Setting memblock NUMA node nid[%d] "
+				"start[%llx] end[%lx]\n",
+				nid, start, this_end);
+
+			memblock_set_node(start, this_end - start,
+					  &memblock.memory, nid);
+			if (memblock.memory.max != prev_max)
+				goto memblock_resized;
+			start = this_end;
+		}
+	}
+}
+
+static int __init grab_mlgroups(struct mdesc_handle *md)
+{
+	unsigned long paddr;
+	int count = 0;
+	u64 node;
+
+	mdesc_for_each_node_by_name(md, node, "memory-latency-group")
+		count++;
+	if (!count)
+		return -ENOENT;
+
+	paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mlgroup),
+				    SMP_CACHE_BYTES);
+	if (!paddr)
+		return -ENOMEM;
+
+	mlgroups = __va(paddr);
+	num_mlgroups = count;
+
+	count = 0;
+	mdesc_for_each_node_by_name(md, node, "memory-latency-group") {
+		struct mdesc_mlgroup *m = &mlgroups[count++];
+		const u64 *val;
+
+		m->node = node;
+
+		val = mdesc_get_property(md, node, "latency", NULL);
+		m->latency = *val;
+		val = mdesc_get_property(md, node, "address-match", NULL);
+		m->match = *val;
+		val = mdesc_get_property(md, node, "address-mask", NULL);
+		m->mask = *val;
+
+		numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "
+			"match[%llx] mask[%llx]\n",
+			count - 1, m->node, m->latency, m->match, m->mask);
+	}
+
+	return 0;
+}
+
+static int __init grab_mblocks(struct mdesc_handle *md)
+{
+	unsigned long paddr;
+	int count = 0;
+	u64 node;
+
+	mdesc_for_each_node_by_name(md, node, "mblock")
+		count++;
+	if (!count)
+		return -ENOENT;
+
+	paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mblock),
+				    SMP_CACHE_BYTES);
+	if (!paddr)
+		return -ENOMEM;
+
+	mblocks = __va(paddr);
+	num_mblocks = count;
+
+	count = 0;
+	mdesc_for_each_node_by_name(md, node, "mblock") {
+		struct mdesc_mblock *m = &mblocks[count++];
+		const u64 *val;
+
+		val = mdesc_get_property(md, node, "base", NULL);
+		m->base = *val;
+		val = mdesc_get_property(md, node, "size", NULL);
+		m->size = *val;
+		val = mdesc_get_property(md, node,
+					 "address-congruence-offset", NULL);
+
+		/* The address-congruence-offset property is optional.
+		 * Explicity zero it be identifty this.
+		 */
+		if (val)
+			m->offset = *val;
+		else
+			m->offset = 0UL;
+
+		numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
+			count - 1, m->base, m->size, m->offset);
+	}
+
+	return 0;
+}
+
+static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,
+					       u64 grp, cpumask_t *mask)
+{
+	u64 arc;
+
+	cpumask_clear(mask);
+
+	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {
+		u64 target = mdesc_arc_target(md, arc);
+		const char *name = mdesc_node_name(md, target);
+		const u64 *id;
+
+		if (strcmp(name, "cpu"))
+			continue;
+		id = mdesc_get_property(md, target, "id", NULL);
+		if (*id < nr_cpu_ids)
+			cpumask_set_cpu(*id, mask);
+	}
+}
+
+static struct mdesc_mlgroup * __init find_mlgroup(u64 node)
+{
+	int i;
+
+	for (i = 0; i < num_mlgroups; i++) {
+		struct mdesc_mlgroup *m = &mlgroups[i];
+		if (m->node == node)
+			return m;
+	}
+	return NULL;
+}
+
+int __node_distance(int from, int to)
+{
+	if ((from >= MAX_NUMNODES) || (to >= MAX_NUMNODES)) {
+		pr_warn("Returning default NUMA distance value for %d->%d\n",
+			from, to);
+		return (from == to) ? LOCAL_DISTANCE : REMOTE_DISTANCE;
+	}
+	return numa_latency[from][to];
+}
+EXPORT_SYMBOL(__node_distance);
+
+static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp)
+{
+	int i;
+
+	for (i = 0; i < MAX_NUMNODES; i++) {
+		struct node_mem_mask *n = &node_masks[i];
+
+		if ((grp->mask == n->mask) && (grp->match == n->match))
+			break;
+	}
+	return i;
+}
+
+static void __init find_numa_latencies_for_group(struct mdesc_handle *md,
+						 u64 grp, int index)
+{
+	u64 arc;
+
+	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
+		int tnode;
+		u64 target = mdesc_arc_target(md, arc);
+		struct mdesc_mlgroup *m = find_mlgroup(target);
+
+		if (!m)
+			continue;
+		tnode = find_best_numa_node_for_mlgroup(m);
+		if (tnode == MAX_NUMNODES)
+			continue;
+		numa_latency[index][tnode] = m->latency;
+	}
+}
+
+static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,
+				      int index)
+{
+	struct mdesc_mlgroup *candidate = NULL;
+	u64 arc, best_latency = ~(u64)0;
+	struct node_mem_mask *n;
+
+	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
+		u64 target = mdesc_arc_target(md, arc);
+		struct mdesc_mlgroup *m = find_mlgroup(target);
+		if (!m)
+			continue;
+		if (m->latency < best_latency) {
+			candidate = m;
+			best_latency = m->latency;
+		}
+	}
+	if (!candidate)
+		return -ENOENT;
+
+	if (num_node_masks != index) {
+		printk(KERN_ERR "Inconsistent NUMA state, "
+		       "index[%d] != num_node_masks[%d]\n",
+		       index, num_node_masks);
+		return -EINVAL;
+	}
+
+	n = &node_masks[num_node_masks++];
+
+	n->mask = candidate->mask;
+	n->match = candidate->match;
+
+	numadbg("NUMA NODE[%d]: mask[%lx] match[%lx] (latency[%llx])\n",
+		index, n->mask, n->match, candidate->latency);
+
+	return 0;
+}
+
+static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,
+					 int index)
+{
+	cpumask_t mask;
+	int cpu;
+
+	numa_parse_mdesc_group_cpus(md, grp, &mask);
+
+	for_each_cpu(cpu, &mask)
+		numa_cpu_lookup_table[cpu] = index;
+	cpumask_copy(&numa_cpumask_lookup_table[index], &mask);
+
+	if (numa_debug) {
+		printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);
+		for_each_cpu(cpu, &mask)
+			printk("%d ", cpu);
+		printk("]\n");
+	}
+
+	return numa_attach_mlgroup(md, grp, index);
+}
+
+static int __init numa_parse_mdesc(void)
+{
+	struct mdesc_handle *md = mdesc_grab();
+	int i, j, err, count;
+	u64 node;
+
+	node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");
+	if (node == MDESC_NODE_NULL) {
+		mdesc_release(md);
+		return -ENOENT;
+	}
+
+	err = grab_mblocks(md);
+	if (err < 0)
+		goto out;
+
+	err = grab_mlgroups(md);
+	if (err < 0)
+		goto out;
+
+	count = 0;
+	mdesc_for_each_node_by_name(md, node, "group") {
+		err = numa_parse_mdesc_group(md, node, count);
+		if (err < 0)
+			break;
+		count++;
+	}
+
+	count = 0;
+	mdesc_for_each_node_by_name(md, node, "group") {
+		find_numa_latencies_for_group(md, node, count);
+		count++;
+	}
+
+	/* Normalize numa latency matrix according to ACPI SLIT spec. */
+	for (i = 0; i < MAX_NUMNODES; i++) {
+		u64 self_latency = numa_latency[i][i];
+
+		for (j = 0; j < MAX_NUMNODES; j++) {
+			numa_latency[i][j] =
+				(numa_latency[i][j] * LOCAL_DISTANCE) /
+				self_latency;
+		}
+	}
+
+	add_node_ranges();
+
+	for (i = 0; i < num_node_masks; i++) {
+		allocate_node_data(i);
+		node_set_online(i);
+	}
+
+	err = 0;
+out:
+	mdesc_release(md);
+	return err;
+}
+
+static int __init numa_parse_jbus(void)
+{
+	unsigned long cpu, index;
+
+	/* NUMA node id is encoded in bits 36 and higher, and there is
+	 * a 1-to-1 mapping from CPU ID to NUMA node ID.
+	 */
+	index = 0;
+	for_each_present_cpu(cpu) {
+		numa_cpu_lookup_table[cpu] = index;
+		cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));
+		node_masks[index].mask = ~((1UL << 36UL) - 1UL);
+		node_masks[index].match = cpu << 36UL;
+
+		index++;
+	}
+	num_node_masks = index;
+
+	add_node_ranges();
+
+	for (index = 0; index < num_node_masks; index++) {
+		allocate_node_data(index);
+		node_set_online(index);
+	}
+
+	return 0;
+}
+
+static int __init numa_parse_sun4u(void)
+{
+	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
+		unsigned long ver;
+
+		__asm__ ("rdpr %%ver, %0" : "=r" (ver));
+		if ((ver >> 32UL) == __JALAPENO_ID ||
+		    (ver >> 32UL) == __SERRANO_ID)
+			return numa_parse_jbus();
+	}
+	return -1;
+}
+
+static int __init bootmem_init_numa(void)
+{
+	int i, j;
+	int err = -1;
+
+	numadbg("bootmem_init_numa()\n");
+
+	/* Some sane defaults for numa latency values */
+	for (i = 0; i < MAX_NUMNODES; i++) {
+		for (j = 0; j < MAX_NUMNODES; j++)
+			numa_latency[i][j] = (i == j) ?
+				LOCAL_DISTANCE : REMOTE_DISTANCE;
+	}
+
+	if (numa_enabled) {
+		if (tlb_type == hypervisor)
+			err = numa_parse_mdesc();
+		else
+			err = numa_parse_sun4u();
+	}
+	return err;
+}
+
+#else
+
+static int bootmem_init_numa(void)
+{
+	return -1;
+}
+
+#endif
+
+static void __init bootmem_init_nonnuma(void)
+{
+	unsigned long top_of_ram = memblock_end_of_DRAM();
+	unsigned long total_ram = memblock_phys_mem_size();
+
+	numadbg("bootmem_init_nonnuma()\n");
+
+	printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
+	       top_of_ram, total_ram);
+	printk(KERN_INFO "Memory hole size: %ldMB\n",
+	       (top_of_ram - total_ram) >> 20);
+
+	init_node_masks_nonnuma();
+	memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0);
+	allocate_node_data(0);
+	node_set_online(0);
+}
+
+static unsigned long __init bootmem_init(unsigned long phys_base)
+{
+	unsigned long end_pfn;
+
+	end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
+	max_pfn = max_low_pfn = end_pfn;
+	min_low_pfn = (phys_base >> PAGE_SHIFT);
+
+	if (bootmem_init_numa() < 0)
+		bootmem_init_nonnuma();
+
+	/* Dump memblock with node info. */
+	memblock_dump_all();
+
+	/* XXX cpu notifier XXX */
+
+	return end_pfn;
+}
+
+static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
+static int pall_ents __initdata;
+
+static unsigned long max_phys_bits = 40;
+
+bool kern_addr_valid(unsigned long addr)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+
+	if ((long)addr < 0L) {
+		unsigned long pa = __pa(addr);
+
+		if ((pa >> max_phys_bits) != 0UL)
+			return false;
+
+		return pfn_valid(pa >> PAGE_SHIFT);
+	}
+
+	if (addr >= (unsigned long) KERNBASE &&
+	    addr < (unsigned long)&_end)
+		return true;
+
+	pgd = pgd_offset_k(addr);
+	if (pgd_none(*pgd))
+		return false;
+
+	p4d = p4d_offset(pgd, addr);
+	if (p4d_none(*p4d))
+		return false;
+
+	pud = pud_offset(p4d, addr);
+	if (pud_none(*pud))
+		return false;
+
+	if (pud_leaf(*pud))
+		return pfn_valid(pud_pfn(*pud));
+
+	pmd = pmd_offset(pud, addr);
+	if (pmd_none(*pmd))
+		return false;
+
+	if (pmd_leaf(*pmd))
+		return pfn_valid(pmd_pfn(*pmd));
+
+	pte = pte_offset_kernel(pmd, addr);
+	if (pte_none(*pte))
+		return false;
+
+	return pfn_valid(pte_pfn(*pte));
+}
+
+static unsigned long __ref kernel_map_hugepud(unsigned long vstart,
+					      unsigned long vend,
+					      pud_t *pud)
+{
+	const unsigned long mask16gb = (1UL << 34) - 1UL;
+	u64 pte_val = vstart;
+
+	/* Each PUD is 8GB */
+	if ((vstart & mask16gb) ||
+	    (vend - vstart <= mask16gb)) {
+		pte_val ^= kern_linear_pte_xor[2];
+		pud_val(*pud) = pte_val | _PAGE_PUD_HUGE;
+
+		return vstart + PUD_SIZE;
+	}
+
+	pte_val ^= kern_linear_pte_xor[3];
+	pte_val |= _PAGE_PUD_HUGE;
+
+	vend = vstart + mask16gb + 1UL;
+	while (vstart < vend) {
+		pud_val(*pud) = pte_val;
+
+		pte_val += PUD_SIZE;
+		vstart += PUD_SIZE;
+		pud++;
+	}
+	return vstart;
+}
+
+static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend,
+				   bool guard)
+{
+	if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE)
+		return true;
+
+	return false;
+}
+
+static unsigned long __ref kernel_map_hugepmd(unsigned long vstart,
+					      unsigned long vend,
+					      pmd_t *pmd)
+{
+	const unsigned long mask256mb = (1UL << 28) - 1UL;
+	const unsigned long mask2gb = (1UL << 31) - 1UL;
+	u64 pte_val = vstart;
+
+	/* Each PMD is 8MB */
+	if ((vstart & mask256mb) ||
+	    (vend - vstart <= mask256mb)) {
+		pte_val ^= kern_linear_pte_xor[0];
+		pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE;
+
+		return vstart + PMD_SIZE;
+	}
+
+	if ((vstart & mask2gb) ||
+	    (vend - vstart <= mask2gb)) {
+		pte_val ^= kern_linear_pte_xor[1];
+		pte_val |= _PAGE_PMD_HUGE;
+		vend = vstart + mask256mb + 1UL;
+	} else {
+		pte_val ^= kern_linear_pte_xor[2];
+		pte_val |= _PAGE_PMD_HUGE;
+		vend = vstart + mask2gb + 1UL;
+	}
+
+	while (vstart < vend) {
+		pmd_val(*pmd) = pte_val;
+
+		pte_val += PMD_SIZE;
+		vstart += PMD_SIZE;
+		pmd++;
+	}
+
+	return vstart;
+}
+
+static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend,
+				   bool guard)
+{
+	if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE)
+		return true;
+
+	return false;
+}
+
+static unsigned long __ref kernel_map_range(unsigned long pstart,
+					    unsigned long pend, pgprot_t prot,
+					    bool use_huge)
+{
+	unsigned long vstart = PAGE_OFFSET + pstart;
+	unsigned long vend = PAGE_OFFSET + pend;
+	unsigned long alloc_bytes = 0UL;
+
+	if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
+		prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
+			    vstart, vend);
+		prom_halt();
+	}
+
+	while (vstart < vend) {
+		unsigned long this_end, paddr = __pa(vstart);
+		pgd_t *pgd = pgd_offset_k(vstart);
+		p4d_t *p4d;
+		pud_t *pud;
+		pmd_t *pmd;
+		pte_t *pte;
+
+		if (pgd_none(*pgd)) {
+			pud_t *new;
+
+			new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+						  PAGE_SIZE);
+			if (!new)
+				goto err_alloc;
+			alloc_bytes += PAGE_SIZE;
+			pgd_populate(&init_mm, pgd, new);
+		}
+
+		p4d = p4d_offset(pgd, vstart);
+		if (p4d_none(*p4d)) {
+			pud_t *new;
+
+			new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+						  PAGE_SIZE);
+			if (!new)
+				goto err_alloc;
+			alloc_bytes += PAGE_SIZE;
+			p4d_populate(&init_mm, p4d, new);
+		}
+
+		pud = pud_offset(p4d, vstart);
+		if (pud_none(*pud)) {
+			pmd_t *new;
+
+			if (kernel_can_map_hugepud(vstart, vend, use_huge)) {
+				vstart = kernel_map_hugepud(vstart, vend, pud);
+				continue;
+			}
+			new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+						  PAGE_SIZE);
+			if (!new)
+				goto err_alloc;
+			alloc_bytes += PAGE_SIZE;
+			pud_populate(&init_mm, pud, new);
+		}
+
+		pmd = pmd_offset(pud, vstart);
+		if (pmd_none(*pmd)) {
+			pte_t *new;
+
+			if (kernel_can_map_hugepmd(vstart, vend, use_huge)) {
+				vstart = kernel_map_hugepmd(vstart, vend, pmd);
+				continue;
+			}
+			new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+						  PAGE_SIZE);
+			if (!new)
+				goto err_alloc;
+			alloc_bytes += PAGE_SIZE;
+			pmd_populate_kernel(&init_mm, pmd, new);
+		}
+
+		pte = pte_offset_kernel(pmd, vstart);
+		this_end = (vstart + PMD_SIZE) & PMD_MASK;
+		if (this_end > vend)
+			this_end = vend;
+
+		while (vstart < this_end) {
+			pte_val(*pte) = (paddr | pgprot_val(prot));
+
+			vstart += PAGE_SIZE;
+			paddr += PAGE_SIZE;
+			pte++;
+		}
+	}
+
+	return alloc_bytes;
+
+err_alloc:
+	panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",
+	      __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+	return -ENOMEM;
+}
+
+static void __init flush_all_kernel_tsbs(void)
+{
+	int i;
+
+	for (i = 0; i < KERNEL_TSB_NENTRIES; i++) {
+		struct tsb *ent = &swapper_tsb[i];
+
+		ent->tag = (1UL << TSB_TAG_INVALID_BIT);
+	}
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) {
+		struct tsb *ent = &swapper_4m_tsb[i];
+
+		ent->tag = (1UL << TSB_TAG_INVALID_BIT);
+	}
+#endif
+}
+
+extern unsigned int kvmap_linear_patch[1];
+
+static void __init kernel_physical_mapping_init(void)
+{
+	unsigned long i, mem_alloced = 0UL;
+	bool use_huge = true;
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+	use_huge = false;
+#endif
+	for (i = 0; i < pall_ents; i++) {
+		unsigned long phys_start, phys_end;
+
+		phys_start = pall[i].phys_addr;
+		phys_end = phys_start + pall[i].reg_size;
+
+		mem_alloced += kernel_map_range(phys_start, phys_end,
+						PAGE_KERNEL, use_huge);
+	}
+
+	printk("Allocated %ld bytes for kernel page tables.\n",
+	       mem_alloced);
+
+	kvmap_linear_patch[0] = 0x01000000; /* nop */
+	flushi(&kvmap_linear_patch[0]);
+
+	flush_all_kernel_tsbs();
+
+	__flush_tlb_all();
+}
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+void __kernel_map_pages(struct page *page, int numpages, int enable)
+{
+	unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
+	unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
+
+	kernel_map_range(phys_start, phys_end,
+			 (enable ? PAGE_KERNEL : __pgprot(0)), false);
+
+	flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
+			       PAGE_OFFSET + phys_end);
+
+	/* we should perform an IPI and flush all tlbs,
+	 * but that can deadlock->flush only current cpu.
+	 */
+	__flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
+				 PAGE_OFFSET + phys_end);
+}
+#endif
+
+unsigned long __init find_ecache_flush_span(unsigned long size)
+{
+	int i;
+
+	for (i = 0; i < pavail_ents; i++) {
+		if (pavail[i].reg_size >= size)
+			return pavail[i].phys_addr;
+	}
+
+	return ~0UL;
+}
+
+unsigned long PAGE_OFFSET;
+EXPORT_SYMBOL(PAGE_OFFSET);
+
+unsigned long VMALLOC_END   = 0x0000010000000000UL;
+EXPORT_SYMBOL(VMALLOC_END);
+
+unsigned long sparc64_va_hole_top =    0xfffff80000000000UL;
+unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL;
+
+static void __init setup_page_offset(void)
+{
+	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
+		/* Cheetah/Panther support a full 64-bit virtual
+		 * address, so we can use all that our page tables
+		 * support.
+		 */
+		sparc64_va_hole_top =    0xfff0000000000000UL;
+		sparc64_va_hole_bottom = 0x0010000000000000UL;
+
+		max_phys_bits = 42;
+	} else if (tlb_type == hypervisor) {
+		switch (sun4v_chip_type) {
+		case SUN4V_CHIP_NIAGARA1:
+		case SUN4V_CHIP_NIAGARA2:
+			/* T1 and T2 support 48-bit virtual addresses.  */
+			sparc64_va_hole_top =    0xffff800000000000UL;
+			sparc64_va_hole_bottom = 0x0000800000000000UL;
+
+			max_phys_bits = 39;
+			break;
+		case SUN4V_CHIP_NIAGARA3:
+			/* T3 supports 48-bit virtual addresses.  */
+			sparc64_va_hole_top =    0xffff800000000000UL;
+			sparc64_va_hole_bottom = 0x0000800000000000UL;
+
+			max_phys_bits = 43;
+			break;
+		case SUN4V_CHIP_NIAGARA4:
+		case SUN4V_CHIP_NIAGARA5:
+		case SUN4V_CHIP_SPARC64X:
+		case SUN4V_CHIP_SPARC_M6:
+			/* T4 and later support 52-bit virtual addresses.  */
+			sparc64_va_hole_top =    0xfff8000000000000UL;
+			sparc64_va_hole_bottom = 0x0008000000000000UL;
+			max_phys_bits = 47;
+			break;
+		case SUN4V_CHIP_SPARC_M7:
+		case SUN4V_CHIP_SPARC_SN:
+			/* M7 and later support 52-bit virtual addresses.  */
+			sparc64_va_hole_top =    0xfff8000000000000UL;
+			sparc64_va_hole_bottom = 0x0008000000000000UL;
+			max_phys_bits = 49;
+			break;
+		case SUN4V_CHIP_SPARC_M8:
+		default:
+			/* M8 and later support 54-bit virtual addresses.
+			 * However, restricting M8 and above VA bits to 53
+			 * as 4-level page table cannot support more than
+			 * 53 VA bits.
+			 */
+			sparc64_va_hole_top =    0xfff0000000000000UL;
+			sparc64_va_hole_bottom = 0x0010000000000000UL;
+			max_phys_bits = 51;
+			break;
+		}
+	}
+
+	if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
+		prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
+			    max_phys_bits);
+		prom_halt();
+	}
+
+	PAGE_OFFSET = sparc64_va_hole_top;
+	VMALLOC_END = ((sparc64_va_hole_bottom >> 1) +
+		       (sparc64_va_hole_bottom >> 2));
+
+	pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
+		PAGE_OFFSET, max_phys_bits);
+	pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n",
+		VMALLOC_START, VMALLOC_END);
+	pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n",
+		VMEMMAP_BASE, VMEMMAP_BASE << 1);
+}
+
+static void __init tsb_phys_patch(void)
+{
+	struct tsb_ldquad_phys_patch_entry *pquad;
+	struct tsb_phys_patch_entry *p;
+
+	pquad = &__tsb_ldquad_phys_patch;
+	while (pquad < &__tsb_ldquad_phys_patch_end) {
+		unsigned long addr = pquad->addr;
+
+		if (tlb_type == hypervisor)
+			*(unsigned int *) addr = pquad->sun4v_insn;
+		else
+			*(unsigned int *) addr = pquad->sun4u_insn;
+		wmb();
+		__asm__ __volatile__("flush	%0"
+				     : /* no outputs */
+				     : "r" (addr));
+
+		pquad++;
+	}
+
+	p = &__tsb_phys_patch;
+	while (p < &__tsb_phys_patch_end) {
+		unsigned long addr = p->addr;
+
+		*(unsigned int *) addr = p->insn;
+		wmb();
+		__asm__ __volatile__("flush	%0"
+				     : /* no outputs */
+				     : "r" (addr));
+
+		p++;
+	}
+}
+
+/* Don't mark as init, we give this to the Hypervisor.  */
+#ifndef CONFIG_DEBUG_PAGEALLOC
+#define NUM_KTSB_DESCR	2
+#else
+#define NUM_KTSB_DESCR	1
+#endif
+static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
+
+/* The swapper TSBs are loaded with a base sequence of:
+ *
+ *	sethi	%uhi(SYMBOL), REG1
+ *	sethi	%hi(SYMBOL), REG2
+ *	or	REG1, %ulo(SYMBOL), REG1
+ *	or	REG2, %lo(SYMBOL), REG2
+ *	sllx	REG1, 32, REG1
+ *	or	REG1, REG2, REG1
+ *
+ * When we use physical addressing for the TSB accesses, we patch the
+ * first four instructions in the above sequence.
+ */
+
+static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
+{
+	unsigned long high_bits, low_bits;
+
+	high_bits = (pa >> 32) & 0xffffffff;
+	low_bits = (pa >> 0) & 0xffffffff;
+
+	while (start < end) {
+		unsigned int *ia = (unsigned int *)(unsigned long)*start;
+
+		ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10);
+		__asm__ __volatile__("flush	%0" : : "r" (ia));
+
+		ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10);
+		__asm__ __volatile__("flush	%0" : : "r" (ia + 1));
+
+		ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff);
+		__asm__ __volatile__("flush	%0" : : "r" (ia + 2));
+
+		ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff);
+		__asm__ __volatile__("flush	%0" : : "r" (ia + 3));
+
+		start++;
+	}
+}
+
+static void ktsb_phys_patch(void)
+{
+	extern unsigned int __swapper_tsb_phys_patch;
+	extern unsigned int __swapper_tsb_phys_patch_end;
+	unsigned long ktsb_pa;
+
+	ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
+	patch_one_ktsb_phys(&__swapper_tsb_phys_patch,
+			    &__swapper_tsb_phys_patch_end, ktsb_pa);
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	{
+	extern unsigned int __swapper_4m_tsb_phys_patch;
+	extern unsigned int __swapper_4m_tsb_phys_patch_end;
+	ktsb_pa = (kern_base +
+		   ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
+	patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,
+			    &__swapper_4m_tsb_phys_patch_end, ktsb_pa);
+	}
+#endif
+}
+
+static void __init sun4v_ktsb_init(void)
+{
+	unsigned long ktsb_pa;
+
+	/* First KTSB for PAGE_SIZE mappings.  */
+	ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
+
+	switch (PAGE_SIZE) {
+	case 8 * 1024:
+	default:
+		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
+		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
+		break;
+
+	case 64 * 1024:
+		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
+		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
+		break;
+
+	case 512 * 1024:
+		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
+		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
+		break;
+
+	case 4 * 1024 * 1024:
+		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
+		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
+		break;
+	}
+
+	ktsb_descr[0].assoc = 1;
+	ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
+	ktsb_descr[0].ctx_idx = 0;
+	ktsb_descr[0].tsb_base = ktsb_pa;
+	ktsb_descr[0].resv = 0;
+
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	/* Second KTSB for 4MB/256MB/2GB/16GB mappings.  */
+	ktsb_pa = (kern_base +
+		   ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
+
+	ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
+	ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |
+				    HV_PGSZ_MASK_256MB |
+				    HV_PGSZ_MASK_2GB |
+				    HV_PGSZ_MASK_16GB) &
+				   cpu_pgsz_mask);
+	ktsb_descr[1].assoc = 1;
+	ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
+	ktsb_descr[1].ctx_idx = 0;
+	ktsb_descr[1].tsb_base = ktsb_pa;
+	ktsb_descr[1].resv = 0;
+#endif
+}
+
+void sun4v_ktsb_register(void)
+{
+	unsigned long pa, ret;
+
+	pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
+
+	ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);
+	if (ret != 0) {
+		prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
+			    "errors with %lx\n", pa, ret);
+		prom_halt();
+	}
+}
+
+static void __init sun4u_linear_pte_xor_finalize(void)
+{
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	/* This is where we would add Panther support for
+	 * 32MB and 256MB pages.
+	 */
+#endif
+}
+
+static void __init sun4v_linear_pte_xor_finalize(void)
+{
+	unsigned long pagecv_flag;
+
+	/* Bit 9 of TTE is no longer CV bit on M7 processor and it instead
+	 * enables MCD error. Do not set bit 9 on M7 processor.
+	 */
+	switch (sun4v_chip_type) {
+	case SUN4V_CHIP_SPARC_M7:
+	case SUN4V_CHIP_SPARC_M8:
+	case SUN4V_CHIP_SPARC_SN:
+		pagecv_flag = 0x00;
+		break;
+	default:
+		pagecv_flag = _PAGE_CV_4V;
+		break;
+	}
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
+		kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
+			PAGE_OFFSET;
+		kern_linear_pte_xor[1] |= (_PAGE_CP_4V | pagecv_flag |
+					   _PAGE_P_4V | _PAGE_W_4V);
+	} else {
+		kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
+	}
+
+	if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
+		kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
+			PAGE_OFFSET;
+		kern_linear_pte_xor[2] |= (_PAGE_CP_4V | pagecv_flag |
+					   _PAGE_P_4V | _PAGE_W_4V);
+	} else {
+		kern_linear_pte_xor[2] = kern_linear_pte_xor[1];
+	}
+
+	if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
+		kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
+			PAGE_OFFSET;
+		kern_linear_pte_xor[3] |= (_PAGE_CP_4V | pagecv_flag |
+					   _PAGE_P_4V | _PAGE_W_4V);
+	} else {
+		kern_linear_pte_xor[3] = kern_linear_pte_xor[2];
+	}
+#endif
+}
+
+/* paging_init() sets up the page tables */
+
+static unsigned long last_valid_pfn;
+
+static void sun4u_pgprot_init(void);
+static void sun4v_pgprot_init(void);
+
+#define _PAGE_CACHE_4U	(_PAGE_CP_4U | _PAGE_CV_4U)
+#define _PAGE_CACHE_4V	(_PAGE_CP_4V | _PAGE_CV_4V)
+#define __DIRTY_BITS_4U	 (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
+#define __DIRTY_BITS_4V	 (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
+#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
+#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
+
+/* We need to exclude reserved regions. This exclusion will include
+ * vmlinux and initrd. To be more precise the initrd size could be used to
+ * compute a new lower limit because it is freed later during initialization.
+ */
+static void __init reduce_memory(phys_addr_t limit_ram)
+{
+	limit_ram += memblock_reserved_size();
+	memblock_enforce_memory_limit(limit_ram);
+}
+
+void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
+{
+	max_zone_pfns[ZONE_NORMAL] = last_valid_pfn;
+}
+
+void __init paging_init(void)
+{
+	unsigned long end_pfn, shift, phys_base;
+	unsigned long real_end, i;
+
+	setup_page_offset();
+
+	/* These build time checkes make sure that the dcache_dirty_cpu()
+	 * folio->flags usage will work.
+	 *
+	 * When a page gets marked as dcache-dirty, we store the
+	 * cpu number starting at bit 32 in the folio->flags.  Also,
+	 * functions like clear_dcache_dirty_cpu use the cpu mask
+	 * in 13-bit signed-immediate instruction fields.
+	 */
+
+	/*
+	 * Page flags must not reach into upper 32 bits that are used
+	 * for the cpu number
+	 */
+	BUILD_BUG_ON(NR_PAGEFLAGS > 32);
+
+	/*
+	 * The bit fields placed in the high range must not reach below
+	 * the 32 bit boundary. Otherwise we cannot place the cpu field
+	 * at the 32 bit boundary.
+	 */
+	BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +
+		ilog2(roundup_pow_of_two(NR_CPUS)) > 32);
+
+	BUILD_BUG_ON(NR_CPUS > 4096);
+
+	kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
+	kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
+
+	/* Invalidate both kernel TSBs.  */
+	memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
+#endif
+
+	/* TTE.cv bit on sparc v9 occupies the same position as TTE.mcde
+	 * bit on M7 processor. This is a conflicting usage of the same
+	 * bit. Enabling TTE.cv on M7 would turn on Memory Corruption
+	 * Detection error on all pages and this will lead to problems
+	 * later. Kernel does not run with MCD enabled and hence rest
+	 * of the required steps to fully configure memory corruption
+	 * detection are not taken. We need to ensure TTE.mcde is not
+	 * set on M7 processor. Compute the value of cacheability
+	 * flag for use later taking this into consideration.
+	 */
+	switch (sun4v_chip_type) {
+	case SUN4V_CHIP_SPARC_M7:
+	case SUN4V_CHIP_SPARC_M8:
+	case SUN4V_CHIP_SPARC_SN:
+		page_cache4v_flag = _PAGE_CP_4V;
+		break;
+	default:
+		page_cache4v_flag = _PAGE_CACHE_4V;
+		break;
+	}
+
+	if (tlb_type == hypervisor)
+		sun4v_pgprot_init();
+	else
+		sun4u_pgprot_init();
+
+	if (tlb_type == cheetah_plus ||
+	    tlb_type == hypervisor) {
+		tsb_phys_patch();
+		ktsb_phys_patch();
+	}
+
+	if (tlb_type == hypervisor)
+		sun4v_patch_tlb_handlers();
+
+	/* Find available physical memory...
+	 *
+	 * Read it twice in order to work around a bug in openfirmware.
+	 * The call to grab this table itself can cause openfirmware to
+	 * allocate memory, which in turn can take away some space from
+	 * the list of available memory.  Reading it twice makes sure
+	 * we really do get the final value.
+	 */
+	read_obp_translations();
+	read_obp_memory("reg", &pall[0], &pall_ents);
+	read_obp_memory("available", &pavail[0], &pavail_ents);
+	read_obp_memory("available", &pavail[0], &pavail_ents);
+
+	phys_base = 0xffffffffffffffffUL;
+	for (i = 0; i < pavail_ents; i++) {
+		phys_base = min(phys_base, pavail[i].phys_addr);
+		memblock_add(pavail[i].phys_addr, pavail[i].reg_size);
+	}
+
+	memblock_reserve(kern_base, kern_size);
+
+	find_ramdisk(phys_base);
+
+	if (cmdline_memory_size)
+		reduce_memory(cmdline_memory_size);
+
+	memblock_allow_resize();
+	memblock_dump_all();
+
+	set_bit(0, mmu_context_bmap);
+
+	shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
+
+	real_end = (unsigned long)_end;
+	num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);
+	printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
+	       num_kernel_image_mappings);
+
+	/* Set kernel pgd to upper alias so physical page computations
+	 * work.
+	 */
+	init_mm.pgd += ((shift) / (sizeof(pgd_t)));
+	
+	memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
+
+	inherit_prom_mappings();
+	
+	/* Ok, we can use our TLB miss and window trap handlers safely.  */
+	setup_tba();
+
+	__flush_tlb_all();
+
+	prom_build_devicetree();
+	of_populate_present_mask();
+#ifndef CONFIG_SMP
+	of_fill_in_cpu_data();
+#endif
+
+	if (tlb_type == hypervisor) {
+		sun4v_mdesc_init();
+		mdesc_populate_present_mask(cpu_all_mask);
+#ifndef CONFIG_SMP
+		mdesc_fill_in_cpu_data(cpu_all_mask);
+#endif
+		mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);
+
+		sun4v_linear_pte_xor_finalize();
+
+		sun4v_ktsb_init();
+		sun4v_ktsb_register();
+	} else {
+		unsigned long impl, ver;
+
+		cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |
+				 HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);
+
+		__asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));
+		impl = ((ver >> 32) & 0xffff);
+		if (impl == PANTHER_IMPL)
+			cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |
+					  HV_PGSZ_MASK_256MB);
+
+		sun4u_linear_pte_xor_finalize();
+	}
+
+	/* Flush the TLBs and the 4M TSB so that the updated linear
+	 * pte XOR settings are realized for all mappings.
+	 */
+	__flush_tlb_all();
+#ifndef CONFIG_DEBUG_PAGEALLOC
+	memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
+#endif
+	__flush_tlb_all();
+
+	/* Setup bootmem... */
+	last_valid_pfn = end_pfn = bootmem_init(phys_base);
+
+	kernel_physical_mapping_init();
+
+	printk("Booting Linux...\n");
+}
+
+int page_in_phys_avail(unsigned long paddr)
+{
+	int i;
+
+	paddr &= PAGE_MASK;
+
+	for (i = 0; i < pavail_ents; i++) {
+		unsigned long start, end;
+
+		start = pavail[i].phys_addr;
+		end = start + pavail[i].reg_size;
+
+		if (paddr >= start && paddr < end)
+			return 1;
+	}
+	if (paddr >= kern_base && paddr < (kern_base + kern_size))
+		return 1;
+#ifdef CONFIG_BLK_DEV_INITRD
+	if (paddr >= __pa(initrd_start) &&
+	    paddr < __pa(PAGE_ALIGN(initrd_end)))
+		return 1;
+#endif
+
+	return 0;
+}
+
+void __init arch_setup_zero_pages(void)
+{
+	phys_addr_t zero_page_pa = kern_base +
+		((unsigned long)&empty_zero_page[0] - KERNBASE);
+
+	__zero_page = phys_to_page(zero_page_pa);
+}
+
+void __init mem_init(void)
+{
+	if (tlb_type == cheetah || tlb_type == cheetah_plus)
+		cheetah_ecache_flush_init();
+}
+
+void free_initmem(void)
+{
+	unsigned long addr, initend;
+	int do_free = 1;
+
+	/* If the physical memory maps were trimmed by kernel command
+	 * line options, don't even try freeing this initmem stuff up.
+	 * The kernel image could have been in the trimmed out region
+	 * and if so the freeing below will free invalid page structs.
+	 */
+	if (cmdline_memory_size)
+		do_free = 0;
+
+	/*
+	 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
+	 */
+	addr = PAGE_ALIGN((unsigned long)(__init_begin));
+	initend = (unsigned long)(__init_end) & PAGE_MASK;
+	for (; addr < initend; addr += PAGE_SIZE) {
+		unsigned long page;
+
+		page = (addr +
+			((unsigned long) __va(kern_base)) -
+			((unsigned long) KERNBASE));
+		memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
+
+		if (do_free)
+			free_reserved_page(virt_to_page(page));
+	}
+}
+
+pgprot_t PAGE_KERNEL __read_mostly;
+EXPORT_SYMBOL(PAGE_KERNEL);
+
+pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
+pgprot_t PAGE_COPY __read_mostly;
+
+pgprot_t PAGE_SHARED __read_mostly;
+EXPORT_SYMBOL(PAGE_SHARED);
+
+unsigned long pg_iobits __read_mostly;
+
+unsigned long _PAGE_IE __read_mostly;
+EXPORT_SYMBOL(_PAGE_IE);
+
+unsigned long _PAGE_E __read_mostly;
+EXPORT_SYMBOL(_PAGE_E);
+
+unsigned long _PAGE_CACHE __read_mostly;
+EXPORT_SYMBOL(_PAGE_CACHE);
+
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
+			       unsigned long addr, unsigned long next)
+{
+	unsigned long pte_base;
+
+	pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
+		    _PAGE_CP_4U | _PAGE_CV_4U |
+		    _PAGE_P_4U | _PAGE_W_4U);
+	if (tlb_type == hypervisor)
+		pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
+			    page_cache4v_flag | _PAGE_P_4V | _PAGE_W_4V);
+
+	pte_base |= _PAGE_PMD_HUGE;
+
+	pmd_val(*pmd) = pte_base | __pa(p);
+}
+
+int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
+			       int node, struct vmem_altmap *altmap)
+{
+	return vmemmap_populate_hugepages(vstart, vend, node, NULL);
+}
+#endif /* CONFIG_SPARSEMEM_VMEMMAP */
+
+/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */
+static pgprot_t protection_map[16] __ro_after_init;
+
+static void prot_init_common(unsigned long page_none,
+			     unsigned long page_shared,
+			     unsigned long page_copy,
+			     unsigned long page_readonly,
+			     unsigned long page_exec_bit)
+{
+	PAGE_COPY = __pgprot(page_copy);
+	PAGE_SHARED = __pgprot(page_shared);
+
+	protection_map[0x0] = __pgprot(page_none);
+	protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
+	protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
+	protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
+	protection_map[0x4] = __pgprot(page_readonly);
+	protection_map[0x5] = __pgprot(page_readonly);
+	protection_map[0x6] = __pgprot(page_copy);
+	protection_map[0x7] = __pgprot(page_copy);
+	protection_map[0x8] = __pgprot(page_none);
+	protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
+	protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
+	protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
+	protection_map[0xc] = __pgprot(page_readonly);
+	protection_map[0xd] = __pgprot(page_readonly);
+	protection_map[0xe] = __pgprot(page_shared);
+	protection_map[0xf] = __pgprot(page_shared);
+}
+
+static void __init sun4u_pgprot_init(void)
+{
+	unsigned long page_none, page_shared, page_copy, page_readonly;
+	unsigned long page_exec_bit;
+	int i;
+
+	PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
+				_PAGE_CACHE_4U | _PAGE_P_4U |
+				__ACCESS_BITS_4U | __DIRTY_BITS_4U |
+				_PAGE_EXEC_4U);
+	PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
+				       _PAGE_CACHE_4U | _PAGE_P_4U |
+				       __ACCESS_BITS_4U | __DIRTY_BITS_4U |
+				       _PAGE_EXEC_4U | _PAGE_L_4U);
+
+	_PAGE_IE = _PAGE_IE_4U;
+	_PAGE_E = _PAGE_E_4U;
+	_PAGE_CACHE = _PAGE_CACHE_4U;
+
+	pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
+		     __ACCESS_BITS_4U | _PAGE_E_4U);
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+	kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
+#else
+	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
+		PAGE_OFFSET;
+#endif
+	kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
+				   _PAGE_P_4U | _PAGE_W_4U);
+
+	for (i = 1; i < 4; i++)
+		kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
+
+	_PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
+			      _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
+			      _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
+
+
+	page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
+	page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
+		       __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
+	page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
+		       __ACCESS_BITS_4U | _PAGE_EXEC_4U);
+	page_readonly   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
+			   __ACCESS_BITS_4U | _PAGE_EXEC_4U);
+
+	page_exec_bit = _PAGE_EXEC_4U;
+
+	prot_init_common(page_none, page_shared, page_copy, page_readonly,
+			 page_exec_bit);
+}
+
+static void __init sun4v_pgprot_init(void)
+{
+	unsigned long page_none, page_shared, page_copy, page_readonly;
+	unsigned long page_exec_bit;
+	int i;
+
+	PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
+				page_cache4v_flag | _PAGE_P_4V |
+				__ACCESS_BITS_4V | __DIRTY_BITS_4V |
+				_PAGE_EXEC_4V);
+	PAGE_KERNEL_LOCKED = PAGE_KERNEL;
+
+	_PAGE_IE = _PAGE_IE_4V;
+	_PAGE_E = _PAGE_E_4V;
+	_PAGE_CACHE = page_cache4v_flag;
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+	kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
+#else
+	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
+		PAGE_OFFSET;
+#endif
+	kern_linear_pte_xor[0] |= (page_cache4v_flag | _PAGE_P_4V |
+				   _PAGE_W_4V);
+
+	for (i = 1; i < 4; i++)
+		kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
+
+	pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
+		     __ACCESS_BITS_4V | _PAGE_E_4V);
+
+	_PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
+			     _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
+			     _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
+			     _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
+
+	page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | page_cache4v_flag;
+	page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
+		       __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
+	page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
+		       __ACCESS_BITS_4V | _PAGE_EXEC_4V);
+	page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
+			 __ACCESS_BITS_4V | _PAGE_EXEC_4V);
+
+	page_exec_bit = _PAGE_EXEC_4V;
+
+	prot_init_common(page_none, page_shared, page_copy, page_readonly,
+			 page_exec_bit);
+}
+
+unsigned long pte_sz_bits(unsigned long sz)
+{
+	if (tlb_type == hypervisor) {
+		switch (sz) {
+		case 8 * 1024:
+		default:
+			return _PAGE_SZ8K_4V;
+		case 64 * 1024:
+			return _PAGE_SZ64K_4V;
+		case 512 * 1024:
+			return _PAGE_SZ512K_4V;
+		case 4 * 1024 * 1024:
+			return _PAGE_SZ4MB_4V;
+		}
+	} else {
+		switch (sz) {
+		case 8 * 1024:
+		default:
+			return _PAGE_SZ8K_4U;
+		case 64 * 1024:
+			return _PAGE_SZ64K_4U;
+		case 512 * 1024:
+			return _PAGE_SZ512K_4U;
+		case 4 * 1024 * 1024:
+			return _PAGE_SZ4MB_4U;
+		}
+	}
+}
+
+pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
+{
+	pte_t pte;
+
+	pte_val(pte)  = page | pgprot_val(pgprot_noncached(prot));
+	pte_val(pte) |= (((unsigned long)space) << 32);
+	pte_val(pte) |= pte_sz_bits(page_size);
+
+	return pte;
+}
+
+static unsigned long kern_large_tte(unsigned long paddr)
+{
+	unsigned long val;
+
+	val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
+	       _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
+	       _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
+	if (tlb_type == hypervisor)
+		val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
+		       page_cache4v_flag | _PAGE_P_4V |
+		       _PAGE_EXEC_4V | _PAGE_W_4V);
+
+	return val | paddr;
+}
+
+/* If not locked, zap it. */
+void __flush_tlb_all(void)
+{
+	unsigned long pstate;
+	int i;
+
+	__asm__ __volatile__("flushw\n\t"
+			     "rdpr	%%pstate, %0\n\t"
+			     "wrpr	%0, %1, %%pstate"
+			     : "=r" (pstate)
+			     : "i" (PSTATE_IE));
+	if (tlb_type == hypervisor) {
+		sun4v_mmu_demap_all();
+	} else if (tlb_type == spitfire) {
+		for (i = 0; i < 64; i++) {
+			/* Spitfire Errata #32 workaround */
+			/* NOTE: Always runs on spitfire, so no
+			 *       cheetah+ page size encodings.
+			 */
+			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
+					     "flush	%%g6"
+					     : /* No outputs */
+					     : "r" (0),
+					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
+
+			if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
+				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+						     "membar #Sync"
+						     : /* no outputs */
+						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
+				spitfire_put_dtlb_data(i, 0x0UL);
+			}
+
+			/* Spitfire Errata #32 workaround */
+			/* NOTE: Always runs on spitfire, so no
+			 *       cheetah+ page size encodings.
+			 */
+			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
+					     "flush	%%g6"
+					     : /* No outputs */
+					     : "r" (0),
+					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
+
+			if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
+				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
+						     "membar #Sync"
+						     : /* no outputs */
+						     : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
+				spitfire_put_itlb_data(i, 0x0UL);
+			}
+		}
+	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
+		cheetah_flush_dtlb_all();
+		cheetah_flush_itlb_all();
+	}
+	__asm__ __volatile__("wrpr	%0, 0, %%pstate"
+			     : : "r" (pstate));
+}
+
+static pte_t *__pte_alloc_one(struct mm_struct *mm)
+{
+	struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0);
+
+	if (!ptdesc)
+		return NULL;
+	if (!pagetable_pte_ctor(mm, ptdesc)) {
+		pagetable_free(ptdesc);
+		return NULL;
+	}
+	return ptdesc_address(ptdesc);
+}
+
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
+{
+	return __pte_alloc_one(mm);
+}
+
+pgtable_t pte_alloc_one(struct mm_struct *mm)
+{
+	return __pte_alloc_one(mm);
+}
+
+static void __pte_free(pgtable_t pte)
+{
+	struct ptdesc *ptdesc = virt_to_ptdesc(pte);
+
+	pagetable_dtor(ptdesc);
+	pagetable_free(ptdesc);
+}
+
+void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+	__pte_free(pte);
+}
+
+void pte_free(struct mm_struct *mm, pgtable_t pte)
+{
+	__pte_free(pte);
+}
+
+void pgtable_free(void *table, bool is_page)
+{
+	if (is_page)
+		__pte_free(table);
+	else
+		kmem_cache_free(pgtable_cache, table);
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static void pte_free_now(struct rcu_head *head)
+{
+	struct page *page;
+
+	page = container_of(head, struct page, rcu_head);
+	__pte_free((pgtable_t)page_address(page));
+}
+
+void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
+{
+	struct page *page;
+
+	page = virt_to_page(pgtable);
+	call_rcu(&page->rcu_head, pte_free_now);
+}
+
+void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
+			  pmd_t *pmd)
+{
+	unsigned long pte, flags;
+	struct mm_struct *mm;
+	pmd_t entry = *pmd;
+
+	if (!pmd_leaf(entry) || !pmd_young(entry))
+		return;
+
+	pte = pmd_val(entry);
+
+	/* Don't insert a non-valid PMD into the TSB, we'll deadlock.  */
+	if (!(pte & _PAGE_VALID))
+		return;
+
+	/* We are fabricating 8MB pages using 4MB real hw pages.  */
+	pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
+
+	mm = vma->vm_mm;
+
+	spin_lock_irqsave(&mm->context.lock, flags);
+
+	if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
+		__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
+					addr, pte);
+
+	spin_unlock_irqrestore(&mm->context.lock, flags);
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+static void context_reload(void *__data)
+{
+	struct mm_struct *mm = __data;
+
+	if (mm == current->mm)
+		load_secondary_context(mm);
+}
+
+void hugetlb_setup(struct pt_regs *regs)
+{
+	struct mm_struct *mm = current->mm;
+	struct tsb_config *tp;
+
+	if (faulthandler_disabled() || !mm) {
+		const struct exception_table_entry *entry;
+
+		entry = search_exception_tables(regs->tpc);
+		if (entry) {
+			regs->tpc = entry->fixup;
+			regs->tnpc = regs->tpc + 4;
+			return;
+		}
+		pr_alert("Unexpected HugeTLB setup in atomic context.\n");
+		die_if_kernel("HugeTSB in atomic", regs);
+	}
+
+	tp = &mm->context.tsb_block[MM_TSB_HUGE];
+	if (likely(tp->tsb == NULL))
+		tsb_grow(mm, MM_TSB_HUGE, 0);
+
+	tsb_context_switch(mm);
+	smp_tsb_sync(mm);
+
+	/* On UltraSPARC-III+ and later, configure the second half of
+	 * the Data-TLB for huge pages.
+	 */
+	if (tlb_type == cheetah_plus) {
+		bool need_context_reload = false;
+		unsigned long ctx;
+
+		spin_lock_irq(&ctx_alloc_lock);
+		ctx = mm->context.sparc64_ctx_val;
+		ctx &= ~CTX_PGSZ_MASK;
+		ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
+		ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
+
+		if (ctx != mm->context.sparc64_ctx_val) {
+			/* When changing the page size fields, we
+			 * must perform a context flush so that no
+			 * stale entries match.  This flush must
+			 * occur with the original context register
+			 * settings.
+			 */
+			do_flush_tlb_mm(mm);
+
+			/* Reload the context register of all processors
+			 * also executing in this address space.
+			 */
+			mm->context.sparc64_ctx_val = ctx;
+			need_context_reload = true;
+		}
+		spin_unlock_irq(&ctx_alloc_lock);
+
+		if (need_context_reload)
+			on_each_cpu(context_reload, mm, 0);
+	}
+}
+#endif
+
+static struct resource code_resource = {
+	.name	= "Kernel code",
+	.flags	= IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
+};
+
+static struct resource data_resource = {
+	.name	= "Kernel data",
+	.flags	= IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
+};
+
+static struct resource bss_resource = {
+	.name	= "Kernel bss",
+	.flags	= IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
+};
+
+static inline resource_size_t compute_kern_paddr(void *addr)
+{
+	return (resource_size_t) (addr - KERNBASE + kern_base);
+}
+
+static void __init kernel_lds_init(void)
+{
+	code_resource.start = compute_kern_paddr(_text);
+	code_resource.end   = compute_kern_paddr(_etext - 1);
+	data_resource.start = compute_kern_paddr(_etext);
+	data_resource.end   = compute_kern_paddr(_edata - 1);
+	bss_resource.start  = compute_kern_paddr(__bss_start);
+	bss_resource.end    = compute_kern_paddr(_end - 1);
+}
+
+static int __init report_memory(void)
+{
+	int i;
+	struct resource *res;
+
+	kernel_lds_init();
+
+	for (i = 0; i < pavail_ents; i++) {
+		res = kzalloc_obj(struct resource);
+
+		if (!res) {
+			pr_warn("Failed to allocate source.\n");
+			break;
+		}
+
+		res->name = "System RAM";
+		res->start = pavail[i].phys_addr;
+		res->end = pavail[i].phys_addr + pavail[i].reg_size - 1;
+		res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
+
+		if (insert_resource(&iomem_resource, res) < 0) {
+			pr_warn("Resource insertion failed.\n");
+			break;
+		}
+
+		insert_resource(res, &code_resource);
+		insert_resource(res, &data_resource);
+		insert_resource(res, &bss_resource);
+	}
+
+	return 0;
+}
+arch_initcall(report_memory);
+
+#ifdef CONFIG_SMP
+#define do_flush_tlb_kernel_range	smp_flush_tlb_kernel_range
+#else
+#define do_flush_tlb_kernel_range	__flush_tlb_kernel_range
+#endif
+
+void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+	if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) {
+		if (start < LOW_OBP_ADDRESS) {
+			flush_tsb_kernel_range(start, LOW_OBP_ADDRESS);
+			do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS);
+		}
+		if (end > HI_OBP_ADDRESS) {
+			flush_tsb_kernel_range(HI_OBP_ADDRESS, end);
+			do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end);
+		}
+	} else {
+		flush_tsb_kernel_range(start, end);
+		do_flush_tlb_kernel_range(start, end);
+	}
+}
+
+void copy_user_highpage(struct page *to, struct page *from,
+	unsigned long vaddr, struct vm_area_struct *vma)
+{
+	char *vfrom, *vto;
+
+	vfrom = kmap_atomic(from);
+	vto = kmap_atomic(to);
+	copy_user_page(vto, vfrom, vaddr, to);
+	kunmap_atomic(vto);
+	kunmap_atomic(vfrom);
+
+	/* If this page has ADI enabled, copy over any ADI tags
+	 * as well
+	 */
+	if (vma->vm_flags & VM_SPARC_ADI) {
+		unsigned long pfrom, pto, i, adi_tag;
+
+		pfrom = page_to_phys(from);
+		pto = page_to_phys(to);
+
+		for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) {
+			asm volatile("ldxa [%1] %2, %0\n\t"
+					: "=r" (adi_tag)
+					:  "r" (i), "i" (ASI_MCD_REAL));
+			asm volatile("stxa %0, [%1] %2\n\t"
+					:
+					: "r" (adi_tag), "r" (pto),
+					  "i" (ASI_MCD_REAL));
+			pto += adi_blksize();
+		}
+		asm volatile("membar #Sync\n\t");
+	}
+}
+EXPORT_SYMBOL(copy_user_highpage);
+
+void copy_highpage(struct page *to, struct page *from)
+{
+	char *vfrom, *vto;
+
+	vfrom = kmap_atomic(from);
+	vto = kmap_atomic(to);
+	copy_page(vto, vfrom);
+	kunmap_atomic(vto);
+	kunmap_atomic(vfrom);
+
+	/* If this platform is ADI enabled, copy any ADI tags
+	 * as well
+	 */
+	if (adi_capable()) {
+		unsigned long pfrom, pto, i, adi_tag;
+
+		pfrom = page_to_phys(from);
+		pto = page_to_phys(to);
+
+		for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) {
+			asm volatile("ldxa [%1] %2, %0\n\t"
+					: "=r" (adi_tag)
+					:  "r" (i), "i" (ASI_MCD_REAL));
+			asm volatile("stxa %0, [%1] %2\n\t"
+					:
+					: "r" (adi_tag), "r" (pto),
+					  "i" (ASI_MCD_REAL));
+			pto += adi_blksize();
+		}
+		asm volatile("membar #Sync\n\t");
+	}
+}
+EXPORT_SYMBOL(copy_highpage);
+
+pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
+{
+	unsigned long prot = pgprot_val(protection_map[vm_flags &
+					(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]);
+
+	if (vm_flags & VM_SPARC_ADI)
+		prot |= _PAGE_MCD_4V;
+
+	return __pgprot(prot);
+}
+EXPORT_SYMBOL(vm_get_page_prot);
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
new file mode 100644
index 000000000..d920a75b5
--- /dev/null
+++ b/arch/sparc/mm/init_64.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _SPARC64_MM_INIT_H
+#define _SPARC64_MM_INIT_H
+
+#include <asm/page.h>
+
+/* Most of the symbols in this file are defined in init.c and
+ * marked non-static so that assembler code can get at them.
+ */
+
+#define MAX_PHYS_ADDRESS	(1UL << MAX_PHYS_ADDRESS_BITS)
+
+extern unsigned long kern_linear_pte_xor[4];
+extern unsigned int sparc64_highest_unlocked_tlb_ent;
+extern unsigned long sparc64_kern_pri_context;
+extern unsigned long sparc64_kern_pri_nuc_bits;
+extern unsigned long sparc64_kern_sec_context;
+void mmu_info(struct seq_file *m);
+
+struct linux_prom_translation {
+	unsigned long virt;
+	unsigned long size;
+	unsigned long data;
+};
+
+/* Exported for kernel TLB miss handling in ktlb.S */
+extern struct linux_prom_translation prom_trans[512];
+extern unsigned int prom_trans_ents;
+
+/* Exported for SMP bootup purposes. */
+extern unsigned long kern_locked_tte_data;
+
+void prom_world(int enter);
+
+#endif /* _SPARC64_MM_INIT_H */
diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c
new file mode 100644
index 000000000..2b5e40cc9
--- /dev/null
+++ b/arch/sparc/mm/io-unit.c
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * io-unit.c:  IO-UNIT specific routines for memory management.
+ *
+ * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
+ */
+ 
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/mm.h>
+#include <linux/bitops.h>
+#include <linux/dma-map-ops.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+#include <asm/io.h>
+#include <asm/io-unit.h>
+#include <asm/mxcc.h>
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+#include <asm/dma.h>
+#include <asm/oplib.h>
+
+#include "mm_32.h"
+
+/* #define IOUNIT_DEBUG */
+#ifdef IOUNIT_DEBUG
+#define IOD(x) printk(x)
+#else
+#define IOD(x) do { } while (0)
+#endif
+
+#define IOPERM        (IOUPTE_CACHE | IOUPTE_WRITE | IOUPTE_VALID)
+#define MKIOPTE(phys) __iopte((((phys)>>4) & IOUPTE_PAGE) | IOPERM)
+
+static const struct dma_map_ops iounit_dma_ops;
+
+static void __init iounit_iommu_init(struct platform_device *op)
+{
+	struct iounit_struct *iounit;
+	iopte_t __iomem *xpt;
+	iopte_t __iomem *xptend;
+
+	iounit = kzalloc_obj(struct iounit_struct, GFP_ATOMIC);
+	if (!iounit) {
+		prom_printf("SUN4D: Cannot alloc iounit, halting.\n");
+		prom_halt();
+	}
+
+	iounit->limit[0] = IOUNIT_BMAP1_START;
+	iounit->limit[1] = IOUNIT_BMAP2_START;
+	iounit->limit[2] = IOUNIT_BMAPM_START;
+	iounit->limit[3] = IOUNIT_BMAPM_END;
+	iounit->rotor[1] = IOUNIT_BMAP2_START;
+	iounit->rotor[2] = IOUNIT_BMAPM_START;
+
+	xpt = of_ioremap(&op->resource[2], 0, PAGE_SIZE * 16, "XPT");
+	if (!xpt) {
+		prom_printf("SUN4D: Cannot map External Page Table.");
+		prom_halt();
+	}
+	
+	op->dev.archdata.iommu = iounit;
+	iounit->page_table = xpt;
+	spin_lock_init(&iounit->lock);
+
+	xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t);
+	for (; xpt < xptend; xpt++)
+		sbus_writel(0, xpt);
+
+	op->dev.dma_ops = &iounit_dma_ops;
+}
+
+static int __init iounit_init(void)
+{
+	extern void sun4d_init_sbi_irq(void);
+	struct device_node *dp;
+
+	for_each_node_by_name(dp, "sbi") {
+		struct platform_device *op = of_find_device_by_node(dp);
+
+		iounit_iommu_init(op);
+		of_propagate_archdata(op);
+	}
+
+	sun4d_init_sbi_irq();
+
+	return 0;
+}
+
+subsys_initcall(iounit_init);
+
+/* One has to hold iounit->lock to call this */
+static dma_addr_t iounit_get_area(struct iounit_struct *iounit,
+				  phys_addr_t phys, int size)
+{
+	int i, j, k, npages;
+	unsigned long rotor, scan, limit;
+	iopte_t iopte;
+
+	npages = (offset_in_page(phys) + size + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+
+	/* A tiny bit of magic ingredience :) */
+	switch (npages) {
+	case 1: i = 0x0231; break;
+	case 2: i = 0x0132; break;
+	default: i = 0x0213; break;
+	}
+	
+	IOD(("%s(%pa,%d[%d])=", __func__, &phys, size, npages));
+	
+next:	j = (i & 15);
+	rotor = iounit->rotor[j - 1];
+	limit = iounit->limit[j];
+	scan = rotor;
+nexti:	scan = find_next_zero_bit(iounit->bmap, limit, scan);
+	if (scan + npages > limit) {
+		if (limit != rotor) {
+			limit = rotor;
+			scan = iounit->limit[j - 1];
+			goto nexti;
+		}
+		i >>= 4;
+		if (!(i & 15))
+			panic("iounit_get_area: Couldn't find free iopte slots for (%pa,%d)\n",
+			      &phys, size);
+		goto next;
+	}
+	for (k = 1, scan++; k < npages; k++)
+		if (test_bit(scan++, iounit->bmap))
+			goto nexti;
+	iounit->rotor[j - 1] = (scan < limit) ? scan : iounit->limit[j - 1];
+	scan -= npages;
+	iopte = MKIOPTE(phys & PAGE_MASK);
+	phys = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + offset_in_page(phys);
+	for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) {
+		set_bit(scan, iounit->bmap);
+		sbus_writel(iopte_val(iopte), &iounit->page_table[scan]);
+	}
+	IOD(("%pa\n", &phys));
+	return phys;
+}
+
+static dma_addr_t iounit_map_phys(struct device *dev, phys_addr_t phys,
+		size_t len, enum dma_data_direction dir, unsigned long attrs)
+{
+	struct iounit_struct *iounit = dev->archdata.iommu;
+	unsigned long flags;
+	dma_addr_t ret;
+	
+	/* XXX So what is maxphys for us and how do drivers know it? */
+	if (!len || len > 256 * 1024)
+		return DMA_MAPPING_ERROR;
+
+	spin_lock_irqsave(&iounit->lock, flags);
+	ret = iounit_get_area(iounit, phys, len);
+	spin_unlock_irqrestore(&iounit->lock, flags);
+	return ret;
+}
+
+static int iounit_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
+		enum dma_data_direction dir, unsigned long attrs)
+{
+	struct iounit_struct *iounit = dev->archdata.iommu;
+	struct scatterlist *sg;
+	unsigned long flags;
+	int i;
+
+	/* FIXME: Cache some resolved pages - often several sg entries are to the same page */
+	spin_lock_irqsave(&iounit->lock, flags);
+	for_each_sg(sgl, sg, nents, i) {
+		sg->dma_address =
+			iounit_get_area(iounit, sg_phys(sg), sg->length);
+		sg->dma_length = sg->length;
+	}
+	spin_unlock_irqrestore(&iounit->lock, flags);
+	return nents;
+}
+
+static void iounit_unmap_phys(struct device *dev, dma_addr_t vaddr, size_t len,
+		enum dma_data_direction dir, unsigned long attrs)
+{
+	struct iounit_struct *iounit = dev->archdata.iommu;
+	unsigned long flags;
+	
+	spin_lock_irqsave(&iounit->lock, flags);
+	len = ((vaddr & ~PAGE_MASK) + len + (PAGE_SIZE-1)) >> PAGE_SHIFT;
+	vaddr = (vaddr - IOUNIT_DMA_BASE) >> PAGE_SHIFT;
+	IOD(("iounit_release %08lx-%08lx\n", (long)vaddr, (long)len+vaddr));
+	for (len += vaddr; vaddr < len; vaddr++)
+		clear_bit(vaddr, iounit->bmap);
+	spin_unlock_irqrestore(&iounit->lock, flags);
+}
+
+static void iounit_unmap_sg(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs)
+{
+	struct iounit_struct *iounit = dev->archdata.iommu;
+	unsigned long flags, vaddr, len;
+	struct scatterlist *sg;
+	int i;
+
+	spin_lock_irqsave(&iounit->lock, flags);
+	for_each_sg(sgl, sg, nents, i) {
+		len = ((sg->dma_address & ~PAGE_MASK) + sg->length + (PAGE_SIZE-1)) >> PAGE_SHIFT;
+		vaddr = (sg->dma_address - IOUNIT_DMA_BASE) >> PAGE_SHIFT;
+		IOD(("iounit_release %08lx-%08lx\n", (long)vaddr, (long)len+vaddr));
+		for (len += vaddr; vaddr < len; vaddr++)
+			clear_bit(vaddr, iounit->bmap);
+	}
+	spin_unlock_irqrestore(&iounit->lock, flags);
+}
+
+#ifdef CONFIG_SBUS
+static void *iounit_alloc(struct device *dev, size_t len,
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
+{
+	struct iounit_struct *iounit = dev->archdata.iommu;
+	unsigned long va, addr, page, end, ret;
+	pgprot_t dvma_prot;
+	iopte_t __iomem *iopte;
+
+	/* XXX So what is maxphys for us and how do drivers know it? */
+	if (!len || len > 256 * 1024)
+		return NULL;
+
+	len = PAGE_ALIGN(len);
+	va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
+	if (!va)
+		return NULL;
+
+	addr = ret = sparc_dma_alloc_resource(dev, len);
+	if (!addr)
+		goto out_free_pages;
+	*dma_handle = addr;
+
+	dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
+	end = PAGE_ALIGN((addr + len));
+	while(addr < end) {
+		page = va;
+		{
+			pmd_t *pmdp;
+			pte_t *ptep;
+			long i;
+
+			pmdp = pmd_off_k(addr);
+			ptep = pte_offset_kernel(pmdp, addr);
+
+			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
+
+			i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT);
+
+			iopte = iounit->page_table + i;
+			sbus_writel(iopte_val(MKIOPTE(__pa(page))), iopte);
+		}
+		addr += PAGE_SIZE;
+		va += PAGE_SIZE;
+	}
+	flush_cache_all();
+	flush_tlb_all();
+
+	return (void *)ret;
+
+out_free_pages:
+	free_pages(va, get_order(len));
+	return NULL;
+}
+
+static void iounit_free(struct device *dev, size_t size, void *cpu_addr,
+		dma_addr_t dma_addr, unsigned long attrs)
+{
+	/* XXX Somebody please fill this in */
+}
+#endif
+
+static const struct dma_map_ops iounit_dma_ops = {
+#ifdef CONFIG_SBUS
+	.alloc			= iounit_alloc,
+	.free			= iounit_free,
+#endif
+	.map_phys		= iounit_map_phys,
+	.unmap_phys		= iounit_unmap_phys,
+	.map_sg			= iounit_map_sg,
+	.unmap_sg		= iounit_unmap_sg,
+};
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
new file mode 100644
index 000000000..df0eb99a4
--- /dev/null
+++ b/arch/sparc/mm/iommu.c
@@ -0,0 +1,457 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * iommu.c:  IOMMU specific routines for memory management.
+ *
+ * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
+ * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
+ * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
+ * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/dma-map-ops.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+#include <asm/io.h>
+#include <asm/mxcc.h>
+#include <asm/mbus.h>
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+#include <asm/bitext.h>
+#include <asm/iommu.h>
+#include <asm/dma.h>
+
+#include "mm_32.h"
+
+/*
+ * This can be sized dynamically, but we will do this
+ * only when we have a guidance about actual I/O pressures.
+ */
+#define IOMMU_RNGE	IOMMU_RNGE_256MB
+#define IOMMU_START	0xF0000000
+#define IOMMU_WINSIZE	(256*1024*1024U)
+#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 256KB */
+#define IOMMU_ORDER	6				/* 4096 * (1<<6) */
+
+static int viking_flush;
+/* viking.S */
+extern void viking_flush_page(unsigned long page);
+extern void viking_mxcc_flush_page(unsigned long page);
+
+/*
+ * Values precomputed according to CPU type.
+ */
+static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
+static pgprot_t dvma_prot;		/* Consistent mapping pte flags */
+
+#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
+#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
+
+static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
+static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
+
+static void __init sbus_iommu_init(struct platform_device *op)
+{
+	struct iommu_struct *iommu;
+	unsigned int impl, vers;
+	unsigned long *bitmap;
+	unsigned long control;
+	unsigned long base;
+	unsigned long tmp;
+
+	iommu = kmalloc_obj(struct iommu_struct);
+	if (!iommu) {
+		prom_printf("Unable to allocate iommu structure\n");
+		prom_halt();
+	}
+
+	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
+				 "iommu_regs");
+	if (!iommu->regs) {
+		prom_printf("Cannot map IOMMU registers\n");
+		prom_halt();
+	}
+
+	control = sbus_readl(&iommu->regs->control);
+	impl = (control & IOMMU_CTRL_IMPL) >> 28;
+	vers = (control & IOMMU_CTRL_VERS) >> 24;
+	control &= ~(IOMMU_CTRL_RNGE);
+	control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
+	sbus_writel(control, &iommu->regs->control);
+
+	iommu_invalidate(iommu->regs);
+	iommu->start = IOMMU_START;
+	iommu->end = 0xffffffff;
+
+	/* Allocate IOMMU page table */
+	/* Stupid alignment constraints give me a headache. 
+	   We need 256K or 512K or 1M or 2M area aligned to
+           its size and current gfp will fortunately give
+           it to us. */
+        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
+	if (!tmp) {
+		prom_printf("Unable to allocate iommu table [0x%lx]\n",
+			    IOMMU_NPTES * sizeof(iopte_t));
+		prom_halt();
+	}
+	iommu->page_table = (iopte_t *)tmp;
+
+	/* Initialize new table. */
+	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
+	flush_cache_all();
+	flush_tlb_all();
+
+	base = __pa((unsigned long)iommu->page_table) >> 4;
+	sbus_writel(base, &iommu->regs->base);
+	iommu_invalidate(iommu->regs);
+
+	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
+	if (!bitmap) {
+		prom_printf("Unable to allocate iommu bitmap [%d]\n",
+			    (int)(IOMMU_NPTES>>3));
+		prom_halt();
+	}
+	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
+	/* To be coherent on HyperSparc, the page color of DVMA
+	 * and physical addresses must match.
+	 */
+	if (srmmu_modtype == HyperSparc)
+		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
+	else
+		iommu->usemap.num_colors = 1;
+
+	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
+	       impl, vers, iommu->page_table,
+	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
+
+	op->dev.archdata.iommu = iommu;
+
+	if (flush_page_for_dma_global)
+		op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
+	 else
+		op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
+}
+
+static int __init iommu_init(void)
+{
+	struct device_node *dp;
+
+	for_each_node_by_name(dp, "iommu") {
+		struct platform_device *op = of_find_device_by_node(dp);
+
+		sbus_iommu_init(op);
+		of_propagate_archdata(op);
+	}
+
+	return 0;
+}
+
+subsys_initcall(iommu_init);
+
+/* Flush the iotlb entries to ram. */
+/* This could be better if we didn't have to flush whole pages. */
+static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
+{
+	unsigned long start;
+	unsigned long end;
+
+	start = (unsigned long)iopte;
+	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
+	start &= PAGE_MASK;
+	if (viking_mxcc_present) {
+		while(start < end) {
+			viking_mxcc_flush_page(start);
+			start += PAGE_SIZE;
+		}
+	} else if (viking_flush) {
+		while(start < end) {
+			viking_flush_page(start);
+			start += PAGE_SIZE;
+		}
+	} else {
+		while(start < end) {
+			__flush_page_to_ram(start);
+			start += PAGE_SIZE;
+		}
+	}
+}
+
+static dma_addr_t __sbus_iommu_map_phys(struct device *dev, phys_addr_t paddr,
+		size_t len, bool per_page_flush, unsigned long attrs)
+{
+	struct iommu_struct *iommu = dev->archdata.iommu;
+	unsigned long off = offset_in_page(paddr);
+	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	unsigned long pfn = __phys_to_pfn(paddr);
+	unsigned int busa, busa0;
+	iopte_t *iopte, *iopte0;
+	int ioptex, i;
+
+	if (unlikely(attrs & DMA_ATTR_MMIO))
+		return DMA_MAPPING_ERROR;
+
+	/* XXX So what is maxphys for us and how do drivers know it? */
+	if (!len || len > 256 * 1024)
+		return DMA_MAPPING_ERROR;
+
+	/*
+	 * We expect unmapped highmem pages to be not in the cache.
+	 * XXX Is this a good assumption?
+	 * XXX What if someone else unmaps it here and races us?
+	 */
+	if (per_page_flush && !PhysHighMem(paddr)) {
+		unsigned long vaddr, p;
+
+		vaddr = (unsigned long)phys_to_virt(paddr);
+		for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
+			flush_page_for_dma(p);
+	}
+
+	/* page color = pfn of page */
+	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
+	if (ioptex < 0)
+		panic("iommu out");
+	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
+	iopte0 = &iommu->page_table[ioptex];
+
+	busa = busa0;
+	iopte = iopte0;
+	for (i = 0; i < npages; i++) {
+		iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
+		iommu_invalidate_page(iommu->regs, busa);
+		busa += PAGE_SIZE;
+		iopte++;
+		pfn++;
+	}
+
+	iommu_flush_iotlb(iopte0, npages);
+	return busa0 + off;
+}
+
+static dma_addr_t sbus_iommu_map_phys_gflush(struct device *dev,
+		phys_addr_t phys, size_t len, enum dma_data_direction dir,
+		unsigned long attrs)
+{
+	flush_page_for_dma(0);
+	return __sbus_iommu_map_phys(dev, phys, len, false, attrs);
+}
+
+static dma_addr_t sbus_iommu_map_phys_pflush(struct device *dev,
+		phys_addr_t phys, size_t len, enum dma_data_direction dir,
+		unsigned long attrs)
+{
+	return __sbus_iommu_map_phys(dev, phys, len, true, attrs);
+}
+
+static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs,
+		bool per_page_flush)
+{
+	struct scatterlist *sg;
+	int j;
+
+	for_each_sg(sgl, sg, nents, j) {
+		sg->dma_address = __sbus_iommu_map_phys(dev, sg_phys(sg),
+				sg->length, per_page_flush, attrs);
+		if (sg->dma_address == DMA_MAPPING_ERROR)
+			return -EIO;
+		sg->dma_length = sg->length;
+	}
+
+	return nents;
+}
+
+static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs)
+{
+	flush_page_for_dma(0);
+	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
+}
+
+static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs)
+{
+	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
+}
+
+static void sbus_iommu_unmap_phys(struct device *dev, dma_addr_t dma_addr,
+		size_t len, enum dma_data_direction dir, unsigned long attrs)
+{
+	struct iommu_struct *iommu = dev->archdata.iommu;
+	unsigned int busa = dma_addr & PAGE_MASK;
+	unsigned long off = dma_addr & ~PAGE_MASK;
+	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
+	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
+	unsigned int i;
+
+	BUG_ON(busa < iommu->start);
+	for (i = 0; i < npages; i++) {
+		iopte_val(iommu->page_table[ioptex + i]) = 0;
+		iommu_invalidate_page(iommu->regs, busa);
+		busa += PAGE_SIZE;
+	}
+	bit_map_clear(&iommu->usemap, ioptex, npages);
+}
+
+static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs)
+{
+	struct scatterlist *sg;
+	int i;
+
+	for_each_sg(sgl, sg, nents, i) {
+		sbus_iommu_unmap_phys(dev, sg->dma_address, sg->length, dir,
+				attrs);
+		sg->dma_address = 0x21212121;
+	}
+}
+
+#ifdef CONFIG_SBUS
+static void *sbus_iommu_alloc(struct device *dev, size_t len,
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
+{
+	struct iommu_struct *iommu = dev->archdata.iommu;
+	unsigned long va, addr, page, end, ret;
+	iopte_t *iopte = iommu->page_table;
+	iopte_t *first;
+	int ioptex;
+
+	/* XXX So what is maxphys for us and how do drivers know it? */
+	if (!len || len > 256 * 1024)
+		return NULL;
+
+	len = PAGE_ALIGN(len);
+	va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
+	if (va == 0)
+		return NULL;
+
+	addr = ret = sparc_dma_alloc_resource(dev, len);
+	if (!addr)
+		goto out_free_pages;
+
+	BUG_ON((va & ~PAGE_MASK) != 0);
+	BUG_ON((addr & ~PAGE_MASK) != 0);
+	BUG_ON((len & ~PAGE_MASK) != 0);
+
+	/* page color = physical address */
+	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
+		addr >> PAGE_SHIFT);
+	if (ioptex < 0)
+		panic("iommu out");
+
+	iopte += ioptex;
+	first = iopte;
+	end = addr + len;
+	while(addr < end) {
+		page = va;
+		{
+			pmd_t *pmdp;
+			pte_t *ptep;
+
+			if (viking_mxcc_present)
+				viking_mxcc_flush_page(page);
+			else if (viking_flush)
+				viking_flush_page(page);
+			else
+				__flush_page_to_ram(page);
+
+			pmdp = pmd_off_k(addr);
+			ptep = pte_offset_kernel(pmdp, addr);
+
+			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
+		}
+		iopte_val(*iopte++) =
+		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
+		addr += PAGE_SIZE;
+		va += PAGE_SIZE;
+	}
+	/* P3: why do we need this?
+	 *
+	 * DAVEM: Because there are several aspects, none of which
+	 *        are handled by a single interface.  Some cpus are
+	 *        completely not I/O DMA coherent, and some have
+	 *        virtually indexed caches.  The driver DMA flushing
+	 *        methods handle the former case, but here during
+	 *        IOMMU page table modifications, and usage of non-cacheable
+	 *        cpu mappings of pages potentially in the cpu caches, we have
+	 *        to handle the latter case as well.
+	 */
+	flush_cache_all();
+	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
+	flush_tlb_all();
+	iommu_invalidate(iommu->regs);
+
+	*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
+	return (void *)ret;
+
+out_free_pages:
+	free_pages(va, get_order(len));
+	return NULL;
+}
+
+static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
+			       dma_addr_t busa, unsigned long attrs)
+{
+	struct iommu_struct *iommu = dev->archdata.iommu;
+	iopte_t *iopte = iommu->page_table;
+	struct page *page = virt_to_page(cpu_addr);
+	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
+	unsigned long end;
+
+	if (!sparc_dma_free_resource(cpu_addr, len))
+		return;
+
+	BUG_ON((busa & ~PAGE_MASK) != 0);
+	BUG_ON((len & ~PAGE_MASK) != 0);
+
+	iopte += ioptex;
+	end = busa + len;
+	while (busa < end) {
+		iopte_val(*iopte++) = 0;
+		busa += PAGE_SIZE;
+	}
+	flush_tlb_all();
+	iommu_invalidate(iommu->regs);
+	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
+
+	__free_pages(page, get_order(len));
+}
+#endif
+
+static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
+#ifdef CONFIG_SBUS
+	.alloc			= sbus_iommu_alloc,
+	.free			= sbus_iommu_free,
+#endif
+	.map_phys		= sbus_iommu_map_phys_gflush,
+	.unmap_phys		= sbus_iommu_unmap_phys,
+	.map_sg			= sbus_iommu_map_sg_gflush,
+	.unmap_sg		= sbus_iommu_unmap_sg,
+};
+
+static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
+#ifdef CONFIG_SBUS
+	.alloc			= sbus_iommu_alloc,
+	.free			= sbus_iommu_free,
+#endif
+	.map_phys		= sbus_iommu_map_phys_pflush,
+	.unmap_phys		= sbus_iommu_unmap_phys,
+	.map_sg			= sbus_iommu_map_sg_pflush,
+	.unmap_sg		= sbus_iommu_unmap_sg,
+};
+
+void __init ld_mmu_iommu(void)
+{
+	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
+		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
+		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
+	} else {
+		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
+		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
+	}
+}
diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c
new file mode 100644
index 000000000..1dc9b3d70
--- /dev/null
+++ b/arch/sparc/mm/leon_mm.c
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  linux/arch/sparc/mm/leon_m.c
+ *
+ * Copyright (C) 2004 Konrad Eisele (eiselekd@web.de, konrad@gaisler.com) Gaisler Research
+ * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB
+ * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB
+ *
+ * do srmmu probe in software
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <asm/asi.h>
+#include <asm/leon.h>
+#include <asm/tlbflush.h>
+
+#include "mm_32.h"
+
+int leon_flush_during_switch = 1;
+static int srmmu_swprobe_trace;
+
+static inline unsigned long leon_get_ctable_ptr(void)
+{
+	unsigned int retval;
+
+	__asm__ __volatile__("lda [%1] %2, %0\n\t" :
+			     "=r" (retval) :
+			     "r" (SRMMU_CTXTBL_PTR),
+			     "i" (ASI_LEON_MMUREGS));
+	return (retval & SRMMU_CTX_PMASK) << 4;
+}
+
+
+unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr)
+{
+
+	unsigned int ctxtbl;
+	unsigned int pgd, pmd, ped;
+	unsigned int ptr;
+	unsigned int lvl, pte;
+	unsigned int ctx;
+	unsigned int paddr_calc;
+
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe: trace on\n");
+
+	ctxtbl = leon_get_ctable_ptr();
+	if (!(ctxtbl)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: leon_get_ctable_ptr returned 0=>0\n");
+		return 0;
+	}
+	if (!_pfn_valid(PFN(ctxtbl))) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO
+			       "swprobe: !_pfn_valid(%x)=>0\n",
+			       PFN(ctxtbl));
+		return 0;
+	}
+
+	ctx = srmmu_get_context();
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe:  --- ctx (%x) ---\n", ctx);
+
+	pgd = LEON_BYPASS_LOAD_PA(ctxtbl + (ctx * 4));
+
+	if (((pgd & SRMMU_ET_MASK) == SRMMU_ET_PTE)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: pgd is entry level 3\n");
+		lvl = 3;
+		pte = pgd;
+		goto ready;
+	}
+	if (((pgd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: pgd is invalid => 0\n");
+		return 0;
+	}
+
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe:  --- pgd (%x) ---\n", pgd);
+
+	ptr = (pgd & SRMMU_PTD_PMASK) << 4;
+	ptr += ((((vaddr) >> LEON_PGD_SH) & LEON_PGD_M) * 4);
+	if (!_pfn_valid(PFN(ptr)))
+		return 0;
+
+	pmd = LEON_BYPASS_LOAD_PA(ptr);
+	if (((pmd & SRMMU_ET_MASK) == SRMMU_ET_PTE)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: pmd is entry level 2\n");
+		lvl = 2;
+		pte = pmd;
+		goto ready;
+	}
+	if (((pmd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: pmd is invalid => 0\n");
+		return 0;
+	}
+
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe:  --- pmd (%x) ---\n", pmd);
+
+	ptr = (pmd & SRMMU_PTD_PMASK) << 4;
+	ptr += (((vaddr >> LEON_PMD_SH) & LEON_PMD_M) * 4);
+	if (!_pfn_valid(PFN(ptr))) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: !_pfn_valid(%x)=>0\n",
+			       PFN(ptr));
+		return 0;
+	}
+
+	ped = LEON_BYPASS_LOAD_PA(ptr);
+
+	if (((ped & SRMMU_ET_MASK) == SRMMU_ET_PTE)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: ped is entry level 1\n");
+		lvl = 1;
+		pte = ped;
+		goto ready;
+	}
+	if (((ped & SRMMU_ET_MASK) != SRMMU_ET_PTD)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: ped is invalid => 0\n");
+		return 0;
+	}
+
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe:  --- ped (%x) ---\n", ped);
+
+	ptr = (ped & SRMMU_PTD_PMASK) << 4;
+	ptr += (((vaddr >> LEON_PTE_SH) & LEON_PTE_M) * 4);
+	if (!_pfn_valid(PFN(ptr)))
+		return 0;
+
+	ptr = LEON_BYPASS_LOAD_PA(ptr);
+	if (((ptr & SRMMU_ET_MASK) == SRMMU_ET_PTE)) {
+		if (srmmu_swprobe_trace)
+			printk(KERN_INFO "swprobe: ptr is entry level 0\n");
+		lvl = 0;
+		pte = ptr;
+		goto ready;
+	}
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe: ptr is invalid => 0\n");
+	return 0;
+
+ready:
+	switch (lvl) {
+	case 0:
+		paddr_calc =
+		    (vaddr & ~(-1 << LEON_PTE_SH)) | ((pte & ~0xff) << 4);
+		break;
+	case 1:
+		paddr_calc =
+		    (vaddr & ~(-1 << LEON_PMD_SH)) | ((pte & ~0xff) << 4);
+		break;
+	case 2:
+		paddr_calc =
+		    (vaddr & ~(-1 << LEON_PGD_SH)) | ((pte & ~0xff) << 4);
+		break;
+	default:
+	case 3:
+		paddr_calc = vaddr;
+		break;
+	}
+	if (srmmu_swprobe_trace)
+		printk(KERN_INFO "swprobe: padde %x\n", paddr_calc);
+	if (paddr)
+		*paddr = paddr_calc;
+	return pte;
+}
+
+void leon_flush_icache_all(void)
+{
+	__asm__ __volatile__(" flush ");	/*iflush*/
+}
+
+void leon_flush_dcache_all(void)
+{
+	__asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" : :
+			     "i"(ASI_LEON_DFLUSH) : "memory");
+}
+
+void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page)
+{
+	if (vma->vm_flags & VM_EXEC)
+		leon_flush_icache_all();
+	leon_flush_dcache_all();
+}
+
+void leon_flush_cache_all(void)
+{
+	__asm__ __volatile__(" flush ");	/*iflush*/
+	__asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" : :
+			     "i"(ASI_LEON_DFLUSH) : "memory");
+}
+
+void leon_flush_tlb_all(void)
+{
+	leon_flush_cache_all();
+	__asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : "r"(0x400),
+			     "i"(ASI_LEON_MMUFLUSH) : "memory");
+}
+
+/* get all cache regs */
+void leon3_getCacheRegs(struct leon3_cacheregs *regs)
+{
+	unsigned long ccr, iccr, dccr;
+
+	if (!regs)
+		return;
+	/* Get Cache regs from "Cache ASI" address 0x0, 0x8 and 0xC */
+	__asm__ __volatile__("lda [%%g0] %3, %0\n\t"
+			     "mov 0x08, %%g1\n\t"
+			     "lda [%%g1] %3, %1\n\t"
+			     "mov 0x0c, %%g1\n\t"
+			     "lda [%%g1] %3, %2\n\t"
+			     : "=r"(ccr), "=r"(iccr), "=r"(dccr)
+			       /* output */
+			     : "i"(ASI_LEON_CACHEREGS)	/* input */
+			     : "g1"	/* clobber list */
+	    );
+	regs->ccr = ccr;
+	regs->iccr = iccr;
+	regs->dccr = dccr;
+}
+
+/* Due to virtual cache we need to check cache configuration if
+ * it is possible to skip flushing in some cases.
+ *
+ * Leon2 and Leon3 differ in their way of telling cache information
+ *
+ */
+int __init leon_flush_needed(void)
+{
+	int flush_needed = -1;
+	unsigned int ssize, sets;
+	char *setStr[4] =
+	    { "direct mapped", "2-way associative", "3-way associative",
+		"4-way associative"
+	};
+	/* leon 3 */
+	struct leon3_cacheregs cregs;
+	leon3_getCacheRegs(&cregs);
+	sets = (cregs.dccr & LEON3_XCCR_SETS_MASK) >> 24;
+	/* (ssize=>realsize) 0=>1k, 1=>2k, 2=>4k, 3=>8k ... */
+	ssize = 1 << ((cregs.dccr & LEON3_XCCR_SSIZE_MASK) >> 20);
+
+	printk(KERN_INFO "CACHE: %s cache, set size %dk\n",
+	       sets > 3 ? "unknown" : setStr[sets], ssize);
+	if ((ssize <= (PAGE_SIZE / 1024)) && (sets == 0)) {
+		/* Set Size <= Page size  ==>
+		   flush on every context switch not needed. */
+		flush_needed = 0;
+		printk(KERN_INFO "CACHE: not flushing on every context switch\n");
+	}
+	return flush_needed;
+}
+
+void leon_switch_mm(void)
+{
+	flush_tlb_mm((void *)0);
+	if (leon_flush_during_switch)
+		leon_flush_cache_all();
+}
+
+static void leon_flush_cache_mm(struct mm_struct *mm)
+{
+	leon_flush_cache_all();
+}
+
+static void leon_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
+{
+	leon_flush_pcache_all(vma, page);
+}
+
+static void leon_flush_cache_range(struct vm_area_struct *vma,
+				   unsigned long start,
+				   unsigned long end)
+{
+	leon_flush_cache_all();
+}
+
+static void leon_flush_tlb_mm(struct mm_struct *mm)
+{
+	leon_flush_tlb_all();
+}
+
+static void leon_flush_tlb_page(struct vm_area_struct *vma,
+				unsigned long page)
+{
+	leon_flush_tlb_all();
+}
+
+static void leon_flush_tlb_range(struct vm_area_struct *vma,
+				 unsigned long start,
+				 unsigned long end)
+{
+	leon_flush_tlb_all();
+}
+
+static void leon_flush_page_to_ram(unsigned long page)
+{
+	leon_flush_cache_all();
+}
+
+static void leon_flush_sig_insns(struct mm_struct *mm, unsigned long page)
+{
+	leon_flush_cache_all();
+}
+
+static void leon_flush_page_for_dma(unsigned long page)
+{
+	leon_flush_dcache_all();
+}
+
+void __init poke_leonsparc(void)
+{
+}
+
+static const struct sparc32_cachetlb_ops leon_ops = {
+	.cache_all	= leon_flush_cache_all,
+	.cache_mm	= leon_flush_cache_mm,
+	.cache_page	= leon_flush_cache_page,
+	.cache_range	= leon_flush_cache_range,
+	.tlb_all	= leon_flush_tlb_all,
+	.tlb_mm		= leon_flush_tlb_mm,
+	.tlb_page	= leon_flush_tlb_page,
+	.tlb_range	= leon_flush_tlb_range,
+	.page_to_ram	= leon_flush_page_to_ram,
+	.sig_insns	= leon_flush_sig_insns,
+	.page_for_dma	= leon_flush_page_for_dma,
+};
+
+void __init init_leon(void)
+{
+	srmmu_name = "LEON";
+	sparc32_cachetlb_ops = &leon_ops;
+	poke_srmmu = poke_leonsparc;
+
+	leon_flush_during_switch = leon_flush_needed();
+}
diff --git a/arch/sparc/mm/mm_32.h b/arch/sparc/mm/mm_32.h
new file mode 100644
index 000000000..ee55f1080
--- /dev/null
+++ b/arch/sparc/mm/mm_32.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* fault_32.c - visible as they are called from assembler */
+asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
+                               unsigned long address);
+
+void window_overflow_fault(void);
+void window_underflow_fault(unsigned long sp);
+void window_ret_fault(struct pt_regs *regs);
+
+/* srmmu.c */
+extern char *srmmu_name;
+extern int viking_mxcc_present;
+extern int flush_page_for_dma_global;
+
+extern void (*poke_srmmu)(void);
+
+void __init srmmu_paging_init(void);
+
+/* iommu.c */
+void ld_mmu_iommu(void);
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
new file mode 100644
index 000000000..9a74902ad
--- /dev/null
+++ b/arch/sparc/mm/srmmu.c
@@ -0,0 +1,1773 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * srmmu.c:  SRMMU specific routines for memory management.
+ *
+ * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
+ * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
+ * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
+ * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
+ * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
+ */
+
+#include <linux/seq_file.h>
+#include <linux/spinlock.h>
+#include <linux/memblock.h>
+#include <linux/pagemap.h>
+#include <linux/vmalloc.h>
+#include <linux/kdebug.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/log2.h>
+#include <linux/gfp.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+
+#include <asm/mmu_context.h>
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+#include <asm/io-unit.h>
+#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
+#include <asm/bitext.h>
+#include <asm/vaddrs.h>
+#include <asm/cache.h>
+#include <asm/traps.h>
+#include <asm/oplib.h>
+#include <asm/mbus.h>
+#include <asm/page.h>
+#include <asm/asi.h>
+#include <asm/smp.h>
+#include <asm/io.h>
+
+/* Now the cpu specific definitions. */
+#include <asm/turbosparc.h>
+#include <asm/tsunami.h>
+#include <asm/viking.h>
+#include <asm/swift.h>
+#include <asm/leon.h>
+#include <asm/mxcc.h>
+#include <asm/ross.h>
+
+#include "mm_32.h"
+
+enum mbus_module srmmu_modtype;
+static unsigned int hwbug_bitmask;
+int vac_cache_size;
+EXPORT_SYMBOL(vac_cache_size);
+int vac_line_size;
+
+extern struct resource sparc_iomap;
+
+extern unsigned long last_valid_pfn;
+
+static pgd_t *srmmu_swapper_pg_dir;
+
+const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops;
+EXPORT_SYMBOL(sparc32_cachetlb_ops);
+
+#ifdef CONFIG_SMP
+const struct sparc32_cachetlb_ops *local_ops;
+
+#define FLUSH_BEGIN(mm)
+#define FLUSH_END
+#else
+#define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) {
+#define FLUSH_END	}
+#endif
+
+int flush_page_for_dma_global = 1;
+
+char *srmmu_name;
+
+ctxd_t *srmmu_ctx_table_phys;
+static ctxd_t *srmmu_context_table;
+
+int viking_mxcc_present;
+static DEFINE_SPINLOCK(srmmu_context_spinlock);
+
+static int is_hypersparc;
+
+static int srmmu_cache_pagetables;
+
+/* these will be initialized in srmmu_nocache_calcsize() */
+static unsigned long srmmu_nocache_size;
+static unsigned long srmmu_nocache_end;
+
+/* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
+#define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
+
+/* The context table is a nocache user with the biggest alignment needs. */
+#define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
+
+void *srmmu_nocache_pool;
+static struct bit_map srmmu_nocache_map;
+
+static inline int srmmu_pmd_none(pmd_t pmd)
+{ return !(pmd_val(pmd) & 0xFFFFFFF); }
+
+/* XXX should we hyper_flush_whole_icache here - Anton */
+static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
+{
+	pte_t pte;
+
+	pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4)));
+	set_pte((pte_t *)ctxp, pte);
+}
+
+/*
+ * Locations of MSI Registers.
+ */
+#define MSI_MBUS_ARBEN	0xe0001008	/* MBus Arbiter Enable register */
+
+/*
+ * Useful bits in the MSI Registers.
+ */
+#define MSI_ASYNC_MODE  0x80000000	/* Operate the MSI asynchronously */
+
+static void msi_set_sync(void)
+{
+	__asm__ __volatile__ ("lda [%0] %1, %%g3\n\t"
+			      "andn %%g3, %2, %%g3\n\t"
+			      "sta %%g3, [%0] %1\n\t" : :
+			      "r" (MSI_MBUS_ARBEN),
+			      "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3");
+}
+
+void pmd_set(pmd_t *pmdp, pte_t *ptep)
+{
+	unsigned long ptp = __nocache_pa(ptep) >> 4;
+	set_pte((pte_t *)&pmd_val(*pmdp), __pte(SRMMU_ET_PTD | ptp));
+}
+
+/*
+ * size: bytes to allocate in the nocache area.
+ * align: bytes, number to align at.
+ * Returns the virtual address of the allocated area.
+ */
+static void *__srmmu_get_nocache(int size, int align)
+{
+	int offset, minsz = 1 << SRMMU_NOCACHE_BITMAP_SHIFT;
+	unsigned long addr;
+
+	if (size < minsz) {
+		printk(KERN_ERR "Size 0x%x too small for nocache request\n",
+		       size);
+		size = minsz;
+	}
+	if (size & (minsz - 1)) {
+		printk(KERN_ERR "Size 0x%x unaligned in nocache request\n",
+		       size);
+		size += minsz - 1;
+	}
+	BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
+
+	offset = bit_map_string_get(&srmmu_nocache_map,
+				    size >> SRMMU_NOCACHE_BITMAP_SHIFT,
+				    align >> SRMMU_NOCACHE_BITMAP_SHIFT);
+	if (offset == -1) {
+		printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
+		       size, (int) srmmu_nocache_size,
+		       srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
+		return NULL;
+	}
+
+	addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
+	return (void *)addr;
+}
+
+void *srmmu_get_nocache(int size, int align)
+{
+	void *tmp;
+
+	tmp = __srmmu_get_nocache(size, align);
+
+	if (tmp)
+		memset(tmp, 0, size);
+
+	return tmp;
+}
+
+void srmmu_free_nocache(void *addr, int size)
+{
+	unsigned long vaddr;
+	int offset;
+
+	vaddr = (unsigned long)addr;
+	if (vaddr < SRMMU_NOCACHE_VADDR) {
+		printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
+		    vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
+		BUG();
+	}
+	if (vaddr + size > srmmu_nocache_end) {
+		printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
+		    vaddr, srmmu_nocache_end);
+		BUG();
+	}
+	if (!is_power_of_2(size)) {
+		printk("Size 0x%x is not a power of 2\n", size);
+		BUG();
+	}
+	if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
+		printk("Size 0x%x is too small\n", size);
+		BUG();
+	}
+	if (vaddr & (size - 1)) {
+		printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
+		BUG();
+	}
+
+	offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
+	size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
+
+	bit_map_clear(&srmmu_nocache_map, offset, size);
+}
+
+static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
+						 unsigned long end);
+
+/* Return how much physical memory we have.  */
+static unsigned long __init probe_memory(void)
+{
+	unsigned long total = 0;
+	int i;
+
+	for (i = 0; sp_banks[i].num_bytes; i++)
+		total += sp_banks[i].num_bytes;
+
+	return total;
+}
+
+/*
+ * Reserve nocache dynamically proportionally to the amount of
+ * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
+ */
+static void __init srmmu_nocache_calcsize(void)
+{
+	unsigned long sysmemavail = probe_memory() / 1024;
+	int srmmu_nocache_npages;
+
+	srmmu_nocache_npages =
+		sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
+
+ /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
+	// if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
+	if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
+		srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
+
+	/* anything above 1280 blows up */
+	if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
+		srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
+
+	srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
+	srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
+}
+
+static void __init srmmu_nocache_init(void)
+{
+	void *srmmu_nocache_bitmap;
+	unsigned int bitmap_bits;
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+	unsigned long paddr, vaddr;
+	unsigned long pteval;
+
+	bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
+
+	srmmu_nocache_pool = memblock_alloc_or_panic(srmmu_nocache_size,
+					    SRMMU_NOCACHE_ALIGN_MAX);
+	memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
+
+	srmmu_nocache_bitmap =
+		memblock_alloc_or_panic(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
+			       SMP_CACHE_BYTES);
+	bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
+
+	srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
+	memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
+	init_mm.pgd = srmmu_swapper_pg_dir;
+
+	srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
+
+	paddr = __pa((unsigned long)srmmu_nocache_pool);
+	vaddr = SRMMU_NOCACHE_VADDR;
+
+	while (vaddr < srmmu_nocache_end) {
+		pgd = pgd_offset_k(vaddr);
+		p4d = p4d_offset(pgd, vaddr);
+		pud = pud_offset(p4d, vaddr);
+		pmd = pmd_offset(__nocache_fix(pud), vaddr);
+		pte = pte_offset_kernel(__nocache_fix(pmd), vaddr);
+
+		pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
+
+		if (srmmu_cache_pagetables)
+			pteval |= SRMMU_CACHE;
+
+		set_pte(__nocache_fix(pte), __pte(pteval));
+
+		vaddr += PAGE_SIZE;
+		paddr += PAGE_SIZE;
+	}
+
+	flush_cache_all();
+	flush_tlb_all();
+}
+
+pgd_t *get_pgd_fast(void)
+{
+	pgd_t *pgd = NULL;
+
+	pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
+	if (pgd) {
+		pgd_t *init = pgd_offset_k(0);
+		memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
+		memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
+						(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
+	}
+
+	return pgd;
+}
+
+/*
+ * Hardware needs alignment to 256 only, but we align to whole page size
+ * to reduce fragmentation problems due to the buddy principle.
+ * XXX Provide actual fragmentation statistics in /proc.
+ *
+ * Alignments up to the page size are the same for physical and virtual
+ * addresses of the nocache area.
+ */
+pgtable_t pte_alloc_one(struct mm_struct *mm)
+{
+	pte_t *ptep;
+	struct page *page;
+
+	if (!(ptep = pte_alloc_one_kernel(mm)))
+		return NULL;
+	page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
+	spin_lock(&mm->page_table_lock);
+	if (page_ref_inc_return(page) == 2 &&
+			!pagetable_pte_ctor(mm, page_ptdesc(page))) {
+		page_ref_dec(page);
+		ptep = NULL;
+	}
+	spin_unlock(&mm->page_table_lock);
+
+	return ptep;
+}
+
+void pte_free(struct mm_struct *mm, pgtable_t ptep)
+{
+	struct page *page;
+
+	page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
+	spin_lock(&mm->page_table_lock);
+	if (page_ref_dec_return(page) == 1)
+		pagetable_dtor(page_ptdesc(page));
+	spin_unlock(&mm->page_table_lock);
+
+	srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE);
+}
+
+/* context handling - a dynamically sized pool is used */
+#define NO_CONTEXT	-1
+
+struct ctx_list {
+	struct ctx_list *next;
+	struct ctx_list *prev;
+	unsigned int ctx_number;
+	struct mm_struct *ctx_mm;
+};
+
+static struct ctx_list *ctx_list_pool;
+static struct ctx_list ctx_free;
+static struct ctx_list ctx_used;
+
+/* At boot time we determine the number of contexts */
+static int num_contexts;
+
+static inline void remove_from_ctx_list(struct ctx_list *entry)
+{
+	entry->next->prev = entry->prev;
+	entry->prev->next = entry->next;
+}
+
+static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
+{
+	entry->next = head;
+	(entry->prev = head->prev)->next = entry;
+	head->prev = entry;
+}
+#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
+#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
+
+
+static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
+{
+	struct ctx_list *ctxp;
+
+	ctxp = ctx_free.next;
+	if (ctxp != &ctx_free) {
+		remove_from_ctx_list(ctxp);
+		add_to_used_ctxlist(ctxp);
+		mm->context = ctxp->ctx_number;
+		ctxp->ctx_mm = mm;
+		return;
+	}
+	ctxp = ctx_used.next;
+	if (ctxp->ctx_mm == old_mm)
+		ctxp = ctxp->next;
+	if (ctxp == &ctx_used)
+		panic("out of mmu contexts");
+	flush_cache_mm(ctxp->ctx_mm);
+	flush_tlb_mm(ctxp->ctx_mm);
+	remove_from_ctx_list(ctxp);
+	add_to_used_ctxlist(ctxp);
+	ctxp->ctx_mm->context = NO_CONTEXT;
+	ctxp->ctx_mm = mm;
+	mm->context = ctxp->ctx_number;
+}
+
+static inline void free_context(int context)
+{
+	struct ctx_list *ctx_old;
+
+	ctx_old = ctx_list_pool + context;
+	remove_from_ctx_list(ctx_old);
+	add_to_free_ctxlist(ctx_old);
+}
+
+static void __init sparc_context_init(int numctx)
+{
+	int ctx;
+	unsigned long size;
+
+	size = numctx * sizeof(struct ctx_list);
+	ctx_list_pool = memblock_alloc_or_panic(size, SMP_CACHE_BYTES);
+
+	for (ctx = 0; ctx < numctx; ctx++) {
+		struct ctx_list *clist;
+
+		clist = (ctx_list_pool + ctx);
+		clist->ctx_number = ctx;
+		clist->ctx_mm = NULL;
+	}
+	ctx_free.next = ctx_free.prev = &ctx_free;
+	ctx_used.next = ctx_used.prev = &ctx_used;
+	for (ctx = 0; ctx < numctx; ctx++)
+		add_to_free_ctxlist(ctx_list_pool + ctx);
+}
+
+void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
+	       struct task_struct *tsk)
+{
+	unsigned long flags;
+
+	if (mm->context == NO_CONTEXT) {
+		spin_lock_irqsave(&srmmu_context_spinlock, flags);
+		alloc_context(old_mm, mm);
+		spin_unlock_irqrestore(&srmmu_context_spinlock, flags);
+		srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
+	}
+
+	if (sparc_cpu_model == sparc_leon)
+		leon_switch_mm();
+
+	if (is_hypersparc)
+		hyper_flush_whole_icache();
+
+	srmmu_set_context(mm->context);
+}
+
+/* Low level IO area allocation on the SRMMU. */
+static inline void srmmu_mapioaddr(unsigned long physaddr,
+				   unsigned long virt_addr, int bus_type)
+{
+	pgd_t *pgdp;
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep;
+	unsigned long tmp;
+
+	physaddr &= PAGE_MASK;
+	pgdp = pgd_offset_k(virt_addr);
+	p4dp = p4d_offset(pgdp, virt_addr);
+	pudp = pud_offset(p4dp, virt_addr);
+	pmdp = pmd_offset(pudp, virt_addr);
+	ptep = pte_offset_kernel(pmdp, virt_addr);
+	tmp = (physaddr >> 4) | SRMMU_ET_PTE;
+
+	/* I need to test whether this is consistent over all
+	 * sun4m's.  The bus_type represents the upper 4 bits of
+	 * 36-bit physical address on the I/O space lines...
+	 */
+	tmp |= (bus_type << 28);
+	tmp |= SRMMU_PRIV;
+	__flush_page_to_ram(virt_addr);
+	set_pte(ptep, __pte(tmp));
+}
+
+void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
+		      unsigned long xva, unsigned int len)
+{
+	while (len != 0) {
+		len -= PAGE_SIZE;
+		srmmu_mapioaddr(xpa, xva, bus);
+		xva += PAGE_SIZE;
+		xpa += PAGE_SIZE;
+	}
+	flush_tlb_all();
+}
+
+static inline void srmmu_unmapioaddr(unsigned long virt_addr)
+{
+	pgd_t *pgdp;
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep;
+
+
+	pgdp = pgd_offset_k(virt_addr);
+	p4dp = p4d_offset(pgdp, virt_addr);
+	pudp = pud_offset(p4dp, virt_addr);
+	pmdp = pmd_offset(pudp, virt_addr);
+	ptep = pte_offset_kernel(pmdp, virt_addr);
+
+	/* No need to flush uncacheable page. */
+	__pte_clear(ptep);
+}
+
+void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
+{
+	while (len != 0) {
+		len -= PAGE_SIZE;
+		srmmu_unmapioaddr(virt_addr);
+		virt_addr += PAGE_SIZE;
+	}
+	flush_tlb_all();
+}
+
+/* tsunami.S */
+extern void tsunami_flush_cache_all(void);
+extern void tsunami_flush_cache_mm(struct mm_struct *mm);
+extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
+extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
+extern void tsunami_flush_page_to_ram(unsigned long page);
+extern void tsunami_flush_page_for_dma(unsigned long page);
+extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
+extern void tsunami_flush_tlb_all(void);
+extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
+extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
+extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
+extern void tsunami_setup_blockops(void);
+
+/* swift.S */
+extern void swift_flush_cache_all(void);
+extern void swift_flush_cache_mm(struct mm_struct *mm);
+extern void swift_flush_cache_range(struct vm_area_struct *vma,
+				    unsigned long start, unsigned long end);
+extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
+extern void swift_flush_page_to_ram(unsigned long page);
+extern void swift_flush_page_for_dma(unsigned long page);
+extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
+extern void swift_flush_tlb_all(void);
+extern void swift_flush_tlb_mm(struct mm_struct *mm);
+extern void swift_flush_tlb_range(struct vm_area_struct *vma,
+				  unsigned long start, unsigned long end);
+extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
+
+
+/*
+ * The following are all MBUS based SRMMU modules, and therefore could
+ * be found in a multiprocessor configuration.  On the whole, these
+ * chips seems to be much more touchy about DVMA and page tables
+ * with respect to cache coherency.
+ */
+
+/* viking.S */
+extern void viking_flush_cache_all(void);
+extern void viking_flush_cache_mm(struct mm_struct *mm);
+extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
+				     unsigned long end);
+extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
+extern void viking_flush_page_to_ram(unsigned long page);
+extern void viking_flush_page_for_dma(unsigned long page);
+extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
+extern void viking_flush_page(unsigned long page);
+extern void viking_mxcc_flush_page(unsigned long page);
+extern void viking_flush_tlb_all(void);
+extern void viking_flush_tlb_mm(struct mm_struct *mm);
+extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+				   unsigned long end);
+extern void viking_flush_tlb_page(struct vm_area_struct *vma,
+				  unsigned long page);
+extern void sun4dsmp_flush_tlb_all(void);
+extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
+extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+				   unsigned long end);
+extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
+				  unsigned long page);
+
+/* hypersparc.S */
+extern void hypersparc_flush_cache_all(void);
+extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
+extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
+extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
+extern void hypersparc_flush_page_to_ram(unsigned long page);
+extern void hypersparc_flush_page_for_dma(unsigned long page);
+extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
+extern void hypersparc_flush_tlb_all(void);
+extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
+extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
+extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
+extern void hypersparc_setup_blockops(void);
+
+/*
+ * NOTE: All of this startup code assumes the low 16mb (approx.) of
+ *       kernel mappings are done with one single contiguous chunk of
+ *       ram.  On small ram machines (classics mainly) we only get
+ *       around 8mb mapped for us.
+ */
+
+static void __init early_pgtable_allocfail(char *type)
+{
+	prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
+	prom_halt();
+}
+
+static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
+							unsigned long end)
+{
+	pgd_t *pgdp;
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep;
+
+	while (start < end) {
+		pgdp = pgd_offset_k(start);
+		p4dp = p4d_offset(pgdp, start);
+		pudp = pud_offset(p4dp, start);
+		if (pud_none(*__nocache_fix(pudp))) {
+			pmdp = __srmmu_get_nocache(
+			    SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
+			if (pmdp == NULL)
+				early_pgtable_allocfail("pmd");
+			memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
+			pud_set(__nocache_fix(pudp), pmdp);
+		}
+		pmdp = pmd_offset(__nocache_fix(pudp), start);
+		if (srmmu_pmd_none(*__nocache_fix(pmdp))) {
+			ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
+			if (ptep == NULL)
+				early_pgtable_allocfail("pte");
+			memset(__nocache_fix(ptep), 0, PTE_SIZE);
+			pmd_set(__nocache_fix(pmdp), ptep);
+		}
+		if (start > (0xffffffffUL - PMD_SIZE))
+			break;
+		start = (start + PMD_SIZE) & PMD_MASK;
+	}
+}
+
+static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
+						  unsigned long end)
+{
+	pgd_t *pgdp;
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep;
+
+	while (start < end) {
+		pgdp = pgd_offset_k(start);
+		p4dp = p4d_offset(pgdp, start);
+		pudp = pud_offset(p4dp, start);
+		if (pud_none(*pudp)) {
+			pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
+			if (pmdp == NULL)
+				early_pgtable_allocfail("pmd");
+			memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
+			pud_set((pud_t *)pgdp, pmdp);
+		}
+		pmdp = pmd_offset(pudp, start);
+		if (srmmu_pmd_none(*pmdp)) {
+			ptep = __srmmu_get_nocache(PTE_SIZE,
+							     PTE_SIZE);
+			if (ptep == NULL)
+				early_pgtable_allocfail("pte");
+			memset(ptep, 0, PTE_SIZE);
+			pmd_set(pmdp, ptep);
+		}
+		if (start > (0xffffffffUL - PMD_SIZE))
+			break;
+		start = (start + PMD_SIZE) & PMD_MASK;
+	}
+}
+
+/* These flush types are not available on all chips... */
+static inline unsigned long srmmu_probe(unsigned long vaddr)
+{
+	unsigned long retval;
+
+	if (sparc_cpu_model != sparc_leon) {
+
+		vaddr &= PAGE_MASK;
+		__asm__ __volatile__("lda [%1] %2, %0\n\t" :
+				     "=r" (retval) :
+				     "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE));
+	} else {
+		retval = leon_swprobe(vaddr, NULL);
+	}
+	return retval;
+}
+
+/*
+ * This is much cleaner than poking around physical address space
+ * looking at the prom's page table directly which is what most
+ * other OS's do.  Yuck... this is much better.
+ */
+static void __init srmmu_inherit_prom_mappings(unsigned long start,
+					       unsigned long end)
+{
+	unsigned long probed;
+	unsigned long addr;
+	pgd_t *pgdp;
+	p4d_t *p4dp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep;
+	int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
+
+	while (start <= end) {
+		if (start == 0)
+			break; /* probably wrap around */
+		if (start == 0xfef00000)
+			start = KADB_DEBUGGER_BEGVM;
+		probed = srmmu_probe(start);
+		if (!probed) {
+			/* continue probing until we find an entry */
+			start += PAGE_SIZE;
+			continue;
+		}
+
+		/* A red snapper, see what it really is. */
+		what = 0;
+		addr = start - PAGE_SIZE;
+
+		if (!(start & ~(PMD_MASK))) {
+			if (srmmu_probe(addr + PMD_SIZE) == probed)
+				what = 1;
+		}
+
+		if (!(start & ~(PGDIR_MASK))) {
+			if (srmmu_probe(addr + PGDIR_SIZE) == probed)
+				what = 2;
+		}
+
+		pgdp = pgd_offset_k(start);
+		p4dp = p4d_offset(pgdp, start);
+		pudp = pud_offset(p4dp, start);
+		if (what == 2) {
+			*__nocache_fix(pgdp) = __pgd(probed);
+			start += PGDIR_SIZE;
+			continue;
+		}
+		if (pud_none(*__nocache_fix(pudp))) {
+			pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
+						   SRMMU_PMD_TABLE_SIZE);
+			if (pmdp == NULL)
+				early_pgtable_allocfail("pmd");
+			memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
+			pud_set(__nocache_fix(pudp), pmdp);
+		}
+		pmdp = pmd_offset(__nocache_fix(pudp), start);
+		if (what == 1) {
+			*(pmd_t *)__nocache_fix(pmdp) = __pmd(probed);
+			start += PMD_SIZE;
+			continue;
+		}
+		if (srmmu_pmd_none(*__nocache_fix(pmdp))) {
+			ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
+			if (ptep == NULL)
+				early_pgtable_allocfail("pte");
+			memset(__nocache_fix(ptep), 0, PTE_SIZE);
+			pmd_set(__nocache_fix(pmdp), ptep);
+		}
+		ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
+		*__nocache_fix(ptep) = __pte(probed);
+		start += PAGE_SIZE;
+	}
+}
+
+#define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
+
+/* Create a third-level SRMMU 16MB page mapping. */
+static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
+{
+	pgd_t *pgdp = pgd_offset_k(vaddr);
+	unsigned long big_pte;
+
+	big_pte = KERNEL_PTE(phys_base >> 4);
+	*__nocache_fix(pgdp) = __pgd(big_pte);
+}
+
+/* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
+static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
+{
+	unsigned long pstart = (sp_banks[sp_entry].base_addr & PGDIR_MASK);
+	unsigned long vstart = (vbase & PGDIR_MASK);
+	unsigned long vend = PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
+	/* Map "low" memory only */
+	const unsigned long min_vaddr = PAGE_OFFSET;
+	const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
+
+	if (vstart < min_vaddr || vstart >= max_vaddr)
+		return vstart;
+
+	if (vend > max_vaddr || vend < min_vaddr)
+		vend = max_vaddr;
+
+	while (vstart < vend) {
+		do_large_mapping(vstart, pstart);
+		vstart += PGDIR_SIZE; pstart += PGDIR_SIZE;
+	}
+	return vstart;
+}
+
+static void __init map_kernel(void)
+{
+	int i;
+
+	if (phys_base > 0) {
+		do_large_mapping(PAGE_OFFSET, phys_base);
+	}
+
+	for (i = 0; sp_banks[i].num_bytes != 0; i++) {
+		map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
+	}
+}
+
+void (*poke_srmmu)(void) = NULL;
+
+void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
+{
+	max_zone_pfns[ZONE_DMA] = max_low_pfn;
+	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
+	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
+}
+
+void __init srmmu_paging_init(void)
+{
+	int i;
+	phandle cpunode;
+	char node_str[128];
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+	unsigned long pages_avail;
+
+	init_mm.context = (unsigned long) NO_CONTEXT;
+	sparc_iomap.start = SUN4M_IOBASE_VADDR;	/* 16MB of IOSPACE on all sun4m's. */
+
+	if (sparc_cpu_model == sun4d)
+		num_contexts = 65536; /* We know it is Viking */
+	else {
+		/* Find the number of contexts on the srmmu. */
+		cpunode = prom_getchild(prom_root_node);
+		num_contexts = 0;
+		while (cpunode != 0) {
+			prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
+			if (!strcmp(node_str, "cpu")) {
+				num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
+				break;
+			}
+			cpunode = prom_getsibling(cpunode);
+		}
+	}
+
+	if (!num_contexts) {
+		prom_printf("Something wrong, can't find cpu node in paging_init.\n");
+		prom_halt();
+	}
+
+	pages_avail = 0;
+	last_valid_pfn = bootmem_init(&pages_avail);
+
+	srmmu_nocache_calcsize();
+	srmmu_nocache_init();
+	srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE));
+	map_kernel();
+
+	/* ctx table has to be physically aligned to its size */
+	srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t));
+	srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table);
+
+	for (i = 0; i < num_contexts; i++)
+		srmmu_ctxd_set(__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
+
+	flush_cache_all();
+	srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
+#ifdef CONFIG_SMP
+	/* Stop from hanging here... */
+	local_ops->tlb_all();
+#else
+	flush_tlb_all();
+#endif
+	poke_srmmu();
+
+	srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
+	srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
+
+	srmmu_allocate_ptable_skeleton(
+		__fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
+	srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
+
+	pgd = pgd_offset_k(PKMAP_BASE);
+	p4d = p4d_offset(pgd, PKMAP_BASE);
+	pud = pud_offset(p4d, PKMAP_BASE);
+	pmd = pmd_offset(pud, PKMAP_BASE);
+	pte = pte_offset_kernel(pmd, PKMAP_BASE);
+	pkmap_page_table = pte;
+
+	flush_cache_all();
+	flush_tlb_all();
+
+	sparc_context_init(num_contexts);
+}
+
+void mmu_info(struct seq_file *m)
+{
+	seq_printf(m,
+		   "MMU type\t: %s\n"
+		   "contexts\t: %d\n"
+		   "nocache total\t: %ld\n"
+		   "nocache used\t: %d\n",
+		   srmmu_name,
+		   num_contexts,
+		   srmmu_nocache_size,
+		   srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
+}
+
+int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+{
+	mm->context = NO_CONTEXT;
+	return 0;
+}
+
+void destroy_context(struct mm_struct *mm)
+{
+	unsigned long flags;
+
+	if (mm->context != NO_CONTEXT) {
+		flush_cache_mm(mm);
+		srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
+		flush_tlb_mm(mm);
+		spin_lock_irqsave(&srmmu_context_spinlock, flags);
+		free_context(mm->context);
+		spin_unlock_irqrestore(&srmmu_context_spinlock, flags);
+		mm->context = NO_CONTEXT;
+	}
+}
+
+/* Init various srmmu chip types. */
+static void __init srmmu_is_bad(void)
+{
+	prom_printf("Could not determine SRMMU chip type.\n");
+	prom_halt();
+}
+
+static void __init init_vac_layout(void)
+{
+	phandle nd;
+	int cache_lines;
+	char node_str[128];
+#ifdef CONFIG_SMP
+	int cpu = 0;
+	unsigned long max_size = 0;
+	unsigned long min_line_size = 0x10000000;
+#endif
+
+	nd = prom_getchild(prom_root_node);
+	while ((nd = prom_getsibling(nd)) != 0) {
+		prom_getstring(nd, "device_type", node_str, sizeof(node_str));
+		if (!strcmp(node_str, "cpu")) {
+			vac_line_size = prom_getint(nd, "cache-line-size");
+			if (vac_line_size == -1) {
+				prom_printf("can't determine cache-line-size, halting.\n");
+				prom_halt();
+			}
+			cache_lines = prom_getint(nd, "cache-nlines");
+			if (cache_lines == -1) {
+				prom_printf("can't determine cache-nlines, halting.\n");
+				prom_halt();
+			}
+
+			vac_cache_size = cache_lines * vac_line_size;
+#ifdef CONFIG_SMP
+			if (vac_cache_size > max_size)
+				max_size = vac_cache_size;
+			if (vac_line_size < min_line_size)
+				min_line_size = vac_line_size;
+			//FIXME: cpus not contiguous!!
+			cpu++;
+			if (cpu >= nr_cpu_ids || !cpu_online(cpu))
+				break;
+#else
+			break;
+#endif
+		}
+	}
+	if (nd == 0) {
+		prom_printf("No CPU nodes found, halting.\n");
+		prom_halt();
+	}
+#ifdef CONFIG_SMP
+	vac_cache_size = max_size;
+	vac_line_size = min_line_size;
+#endif
+	printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
+	       (int)vac_cache_size, (int)vac_line_size);
+}
+
+static void poke_hypersparc(void)
+{
+	volatile unsigned long clear;
+	unsigned long mreg = srmmu_get_mmureg();
+
+	hyper_flush_unconditional_combined();
+
+	mreg &= ~(HYPERSPARC_CWENABLE);
+	mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
+	mreg |= (HYPERSPARC_CMODE);
+
+	srmmu_set_mmureg(mreg);
+
+#if 0 /* XXX I think this is bad news... -DaveM */
+	hyper_clear_all_tags();
+#endif
+
+	put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
+	hyper_flush_whole_icache();
+	clear = srmmu_get_faddr();
+	clear = srmmu_get_fstatus();
+}
+
+static const struct sparc32_cachetlb_ops hypersparc_ops = {
+	.cache_all	= hypersparc_flush_cache_all,
+	.cache_mm	= hypersparc_flush_cache_mm,
+	.cache_page	= hypersparc_flush_cache_page,
+	.cache_range	= hypersparc_flush_cache_range,
+	.tlb_all	= hypersparc_flush_tlb_all,
+	.tlb_mm		= hypersparc_flush_tlb_mm,
+	.tlb_page	= hypersparc_flush_tlb_page,
+	.tlb_range	= hypersparc_flush_tlb_range,
+	.page_to_ram	= hypersparc_flush_page_to_ram,
+	.sig_insns	= hypersparc_flush_sig_insns,
+	.page_for_dma	= hypersparc_flush_page_for_dma,
+};
+
+static void __init init_hypersparc(void)
+{
+	srmmu_name = "ROSS HyperSparc";
+	srmmu_modtype = HyperSparc;
+
+	init_vac_layout();
+
+	is_hypersparc = 1;
+	sparc32_cachetlb_ops = &hypersparc_ops;
+
+	poke_srmmu = poke_hypersparc;
+
+	hypersparc_setup_blockops();
+}
+
+static void poke_swift(void)
+{
+	unsigned long mreg;
+
+	/* Clear any crap from the cache or else... */
+	swift_flush_cache_all();
+
+	/* Enable I & D caches */
+	mreg = srmmu_get_mmureg();
+	mreg |= (SWIFT_IE | SWIFT_DE);
+	/*
+	 * The Swift branch folding logic is completely broken.  At
+	 * trap time, if things are just right, if can mistakenly
+	 * think that a trap is coming from kernel mode when in fact
+	 * it is coming from user mode (it mis-executes the branch in
+	 * the trap code).  So you see things like crashme completely
+	 * hosing your machine which is completely unacceptable.  Turn
+	 * this shit off... nice job Fujitsu.
+	 */
+	mreg &= ~(SWIFT_BF);
+	srmmu_set_mmureg(mreg);
+}
+
+static const struct sparc32_cachetlb_ops swift_ops = {
+	.cache_all	= swift_flush_cache_all,
+	.cache_mm	= swift_flush_cache_mm,
+	.cache_page	= swift_flush_cache_page,
+	.cache_range	= swift_flush_cache_range,
+	.tlb_all	= swift_flush_tlb_all,
+	.tlb_mm		= swift_flush_tlb_mm,
+	.tlb_page	= swift_flush_tlb_page,
+	.tlb_range	= swift_flush_tlb_range,
+	.page_to_ram	= swift_flush_page_to_ram,
+	.sig_insns	= swift_flush_sig_insns,
+	.page_for_dma	= swift_flush_page_for_dma,
+};
+
+#define SWIFT_MASKID_ADDR  0x10003018
+static void __init init_swift(void)
+{
+	unsigned long swift_rev;
+
+	__asm__ __volatile__("lda [%1] %2, %0\n\t"
+			     "srl %0, 0x18, %0\n\t" :
+			     "=r" (swift_rev) :
+			     "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
+	srmmu_name = "Fujitsu Swift";
+	switch (swift_rev) {
+	case 0x11:
+	case 0x20:
+	case 0x23:
+	case 0x30:
+		srmmu_modtype = Swift_lots_o_bugs;
+		hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
+		/*
+		 * Gee george, I wonder why Sun is so hush hush about
+		 * this hardware bug... really braindamage stuff going
+		 * on here.  However I think we can find a way to avoid
+		 * all of the workaround overhead under Linux.  Basically,
+		 * any page fault can cause kernel pages to become user
+		 * accessible (the mmu gets confused and clears some of
+		 * the ACC bits in kernel ptes).  Aha, sounds pretty
+		 * horrible eh?  But wait, after extensive testing it appears
+		 * that if you use pgd_t level large kernel pte's (like the
+		 * 4MB pages on the Pentium) the bug does not get tripped
+		 * at all.  This avoids almost all of the major overhead.
+		 * Welcome to a world where your vendor tells you to,
+		 * "apply this kernel patch" instead of "sorry for the
+		 * broken hardware, send it back and we'll give you
+		 * properly functioning parts"
+		 */
+		break;
+	case 0x25:
+	case 0x31:
+		srmmu_modtype = Swift_bad_c;
+		hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
+		/*
+		 * You see Sun allude to this hardware bug but never
+		 * admit things directly, they'll say things like,
+		 * "the Swift chip cache problems" or similar.
+		 */
+		break;
+	default:
+		srmmu_modtype = Swift_ok;
+		break;
+	}
+
+	sparc32_cachetlb_ops = &swift_ops;
+	flush_page_for_dma_global = 0;
+
+	/*
+	 * Are you now convinced that the Swift is one of the
+	 * biggest VLSI abortions of all time?  Bravo Fujitsu!
+	 * Fujitsu, the !#?!%$'d up processor people.  I bet if
+	 * you examined the microcode of the Swift you'd find
+	 * XXX's all over the place.
+	 */
+	poke_srmmu = poke_swift;
+}
+
+static void turbosparc_flush_cache_all(void)
+{
+	flush_user_windows();
+	turbosparc_idflash_clear();
+}
+
+static void turbosparc_flush_cache_mm(struct mm_struct *mm)
+{
+	FLUSH_BEGIN(mm)
+	flush_user_windows();
+	turbosparc_idflash_clear();
+	FLUSH_END
+}
+
+static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
+{
+	FLUSH_BEGIN(vma->vm_mm)
+	flush_user_windows();
+	turbosparc_idflash_clear();
+	FLUSH_END
+}
+
+static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
+{
+	FLUSH_BEGIN(vma->vm_mm)
+	flush_user_windows();
+	if (vma->vm_flags & VM_EXEC)
+		turbosparc_flush_icache();
+	turbosparc_flush_dcache();
+	FLUSH_END
+}
+
+/* TurboSparc is copy-back, if we turn it on, but this does not work. */
+static void turbosparc_flush_page_to_ram(unsigned long page)
+{
+#ifdef TURBOSPARC_WRITEBACK
+	volatile unsigned long clear;
+
+	if (srmmu_probe(page))
+		turbosparc_flush_page_cache(page);
+	clear = srmmu_get_fstatus();
+#endif
+}
+
+static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
+{
+}
+
+static void turbosparc_flush_page_for_dma(unsigned long page)
+{
+	turbosparc_flush_dcache();
+}
+
+static void turbosparc_flush_tlb_all(void)
+{
+	srmmu_flush_whole_tlb();
+}
+
+static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
+{
+	FLUSH_BEGIN(mm)
+	srmmu_flush_whole_tlb();
+	FLUSH_END
+}
+
+static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
+{
+	FLUSH_BEGIN(vma->vm_mm)
+	srmmu_flush_whole_tlb();
+	FLUSH_END
+}
+
+static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
+{
+	FLUSH_BEGIN(vma->vm_mm)
+	srmmu_flush_whole_tlb();
+	FLUSH_END
+}
+
+
+static void poke_turbosparc(void)
+{
+	unsigned long mreg = srmmu_get_mmureg();
+	unsigned long ccreg;
+
+	/* Clear any crap from the cache or else... */
+	turbosparc_flush_cache_all();
+	/* Temporarily disable I & D caches */
+	mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE);
+	mreg &= ~(TURBOSPARC_PCENABLE);		/* Don't check parity */
+	srmmu_set_mmureg(mreg);
+
+	ccreg = turbosparc_get_ccreg();
+
+#ifdef TURBOSPARC_WRITEBACK
+	ccreg |= (TURBOSPARC_SNENABLE);		/* Do DVMA snooping in Dcache */
+	ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
+			/* Write-back D-cache, emulate VLSI
+			 * abortion number three, not number one */
+#else
+	/* For now let's play safe, optimize later */
+	ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
+			/* Do DVMA snooping in Dcache, Write-thru D-cache */
+	ccreg &= ~(TURBOSPARC_uS2);
+			/* Emulate VLSI abortion number three, not number one */
+#endif
+
+	switch (ccreg & 7) {
+	case 0: /* No SE cache */
+	case 7: /* Test mode */
+		break;
+	default:
+		ccreg |= (TURBOSPARC_SCENABLE);
+	}
+	turbosparc_set_ccreg(ccreg);
+
+	mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
+	mreg |= (TURBOSPARC_ICSNOOP);		/* Icache snooping on */
+	srmmu_set_mmureg(mreg);
+}
+
+static const struct sparc32_cachetlb_ops turbosparc_ops = {
+	.cache_all	= turbosparc_flush_cache_all,
+	.cache_mm	= turbosparc_flush_cache_mm,
+	.cache_page	= turbosparc_flush_cache_page,
+	.cache_range	= turbosparc_flush_cache_range,
+	.tlb_all	= turbosparc_flush_tlb_all,
+	.tlb_mm		= turbosparc_flush_tlb_mm,
+	.tlb_page	= turbosparc_flush_tlb_page,
+	.tlb_range	= turbosparc_flush_tlb_range,
+	.page_to_ram	= turbosparc_flush_page_to_ram,
+	.sig_insns	= turbosparc_flush_sig_insns,
+	.page_for_dma	= turbosparc_flush_page_for_dma,
+};
+
+static void __init init_turbosparc(void)
+{
+	srmmu_name = "Fujitsu TurboSparc";
+	srmmu_modtype = TurboSparc;
+	sparc32_cachetlb_ops = &turbosparc_ops;
+	poke_srmmu = poke_turbosparc;
+}
+
+static void poke_tsunami(void)
+{
+	unsigned long mreg = srmmu_get_mmureg();
+
+	tsunami_flush_icache();
+	tsunami_flush_dcache();
+	mreg &= ~TSUNAMI_ITD;
+	mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
+	srmmu_set_mmureg(mreg);
+}
+
+static const struct sparc32_cachetlb_ops tsunami_ops = {
+	.cache_all	= tsunami_flush_cache_all,
+	.cache_mm	= tsunami_flush_cache_mm,
+	.cache_page	= tsunami_flush_cache_page,
+	.cache_range	= tsunami_flush_cache_range,
+	.tlb_all	= tsunami_flush_tlb_all,
+	.tlb_mm		= tsunami_flush_tlb_mm,
+	.tlb_page	= tsunami_flush_tlb_page,
+	.tlb_range	= tsunami_flush_tlb_range,
+	.page_to_ram	= tsunami_flush_page_to_ram,
+	.sig_insns	= tsunami_flush_sig_insns,
+	.page_for_dma	= tsunami_flush_page_for_dma,
+};
+
+static void __init init_tsunami(void)
+{
+	/*
+	 * Tsunami's pretty sane, Sun and TI actually got it
+	 * somewhat right this time.  Fujitsu should have
+	 * taken some lessons from them.
+	 */
+
+	srmmu_name = "TI Tsunami";
+	srmmu_modtype = Tsunami;
+	sparc32_cachetlb_ops = &tsunami_ops;
+	poke_srmmu = poke_tsunami;
+
+	tsunami_setup_blockops();
+}
+
+static void poke_viking(void)
+{
+	unsigned long mreg = srmmu_get_mmureg();
+	static int smp_catch;
+
+	if (viking_mxcc_present) {
+		unsigned long mxcc_control = mxcc_get_creg();
+
+		mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
+		mxcc_control &= ~(MXCC_CTL_RRC);
+		mxcc_set_creg(mxcc_control);
+
+		/*
+		 * We don't need memory parity checks.
+		 * XXX This is a mess, have to dig out later. ecd.
+		viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
+		 */
+
+		/* We do cache ptables on MXCC. */
+		mreg |= VIKING_TCENABLE;
+	} else {
+		unsigned long bpreg;
+
+		mreg &= ~(VIKING_TCENABLE);
+		if (smp_catch++) {
+			/* Must disable mixed-cmd mode here for other cpu's. */
+			bpreg = viking_get_bpreg();
+			bpreg &= ~(VIKING_ACTION_MIX);
+			viking_set_bpreg(bpreg);
+
+			/* Just in case PROM does something funny. */
+			msi_set_sync();
+		}
+	}
+
+	mreg |= VIKING_SPENABLE;
+	mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
+	mreg |= VIKING_SBENABLE;
+	mreg &= ~(VIKING_ACENABLE);
+	srmmu_set_mmureg(mreg);
+}
+
+static struct sparc32_cachetlb_ops viking_ops __ro_after_init = {
+	.cache_all	= viking_flush_cache_all,
+	.cache_mm	= viking_flush_cache_mm,
+	.cache_page	= viking_flush_cache_page,
+	.cache_range	= viking_flush_cache_range,
+	.tlb_all	= viking_flush_tlb_all,
+	.tlb_mm		= viking_flush_tlb_mm,
+	.tlb_page	= viking_flush_tlb_page,
+	.tlb_range	= viking_flush_tlb_range,
+	.page_to_ram	= viking_flush_page_to_ram,
+	.sig_insns	= viking_flush_sig_insns,
+	.page_for_dma	= viking_flush_page_for_dma,
+};
+
+#ifdef CONFIG_SMP
+/* On sun4d the cpu broadcasts local TLB flushes, so we can just
+ * perform the local TLB flush and all the other cpus will see it.
+ * But, unfortunately, there is a bug in the sun4d XBUS backplane
+ * that requires that we add some synchronization to these flushes.
+ *
+ * The bug is that the fifo which keeps track of all the pending TLB
+ * broadcasts in the system is an entry or two too small, so if we
+ * have too many going at once we'll overflow that fifo and lose a TLB
+ * flush resulting in corruption.
+ *
+ * Our workaround is to take a global spinlock around the TLB flushes,
+ * which guarentees we won't ever have too many pending.  It's a big
+ * hammer, but a semaphore like system to make sure we only have N TLB
+ * flushes going at once will require SMP locking anyways so there's
+ * no real value in trying any harder than this.
+ */
+static struct sparc32_cachetlb_ops viking_sun4d_smp_ops __ro_after_init = {
+	.cache_all	= viking_flush_cache_all,
+	.cache_mm	= viking_flush_cache_mm,
+	.cache_page	= viking_flush_cache_page,
+	.cache_range	= viking_flush_cache_range,
+	.tlb_all	= sun4dsmp_flush_tlb_all,
+	.tlb_mm		= sun4dsmp_flush_tlb_mm,
+	.tlb_page	= sun4dsmp_flush_tlb_page,
+	.tlb_range	= sun4dsmp_flush_tlb_range,
+	.page_to_ram	= viking_flush_page_to_ram,
+	.sig_insns	= viking_flush_sig_insns,
+	.page_for_dma	= viking_flush_page_for_dma,
+};
+#endif
+
+static void __init init_viking(void)
+{
+	unsigned long mreg = srmmu_get_mmureg();
+
+	/* Ahhh, the viking.  SRMMU VLSI abortion number two... */
+	if (mreg & VIKING_MMODE) {
+		srmmu_name = "TI Viking";
+		viking_mxcc_present = 0;
+		msi_set_sync();
+
+		/*
+		 * We need this to make sure old viking takes no hits
+		 * on its cache for dma snoops to workaround the
+		 * "load from non-cacheable memory" interrupt bug.
+		 * This is only necessary because of the new way in
+		 * which we use the IOMMU.
+		 */
+		viking_ops.page_for_dma = viking_flush_page;
+#ifdef CONFIG_SMP
+		viking_sun4d_smp_ops.page_for_dma = viking_flush_page;
+#endif
+		flush_page_for_dma_global = 0;
+	} else {
+		srmmu_name = "TI Viking/MXCC";
+		viking_mxcc_present = 1;
+		srmmu_cache_pagetables = 1;
+	}
+
+	sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
+		&viking_ops;
+#ifdef CONFIG_SMP
+	if (sparc_cpu_model == sun4d)
+		sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
+			&viking_sun4d_smp_ops;
+#endif
+
+	poke_srmmu = poke_viking;
+}
+
+/* Probe for the srmmu chip version. */
+static void __init get_srmmu_type(void)
+{
+	unsigned long mreg, psr;
+	unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
+
+	srmmu_modtype = SRMMU_INVAL_MOD;
+	hwbug_bitmask = 0;
+
+	mreg = srmmu_get_mmureg(); psr = get_psr();
+	mod_typ = (mreg & 0xf0000000) >> 28;
+	mod_rev = (mreg & 0x0f000000) >> 24;
+	psr_typ = (psr >> 28) & 0xf;
+	psr_vers = (psr >> 24) & 0xf;
+
+	/* First, check for sparc-leon. */
+	if (sparc_cpu_model == sparc_leon) {
+		init_leon();
+		return;
+	}
+
+	/* Second, check for HyperSparc or Cypress. */
+	if (mod_typ == 1) {
+		switch (mod_rev) {
+		case 7:
+			/* UP or MP Hypersparc */
+			init_hypersparc();
+			break;
+		case 0:
+		case 2:
+		case 10:
+		case 11:
+		case 12:
+		case 13:
+		case 14:
+		case 15:
+		default:
+			prom_printf("Sparc-Linux Cypress support does not longer exit.\n");
+			prom_halt();
+			break;
+		}
+		return;
+	}
+
+	/* Now Fujitsu TurboSparc. It might happen that it is
+	 * in Swift emulation mode, so we will check later...
+	 */
+	if (psr_typ == 0 && psr_vers == 5) {
+		init_turbosparc();
+		return;
+	}
+
+	/* Next check for Fujitsu Swift. */
+	if (psr_typ == 0 && psr_vers == 4) {
+		phandle cpunode;
+		char node_str[128];
+
+		/* Look if it is not a TurboSparc emulating Swift... */
+		cpunode = prom_getchild(prom_root_node);
+		while ((cpunode = prom_getsibling(cpunode)) != 0) {
+			prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
+			if (!strcmp(node_str, "cpu")) {
+				if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
+				    prom_getintdefault(cpunode, "psr-version", 1) == 5) {
+					init_turbosparc();
+					return;
+				}
+				break;
+			}
+		}
+
+		init_swift();
+		return;
+	}
+
+	/* Now the Viking family of srmmu. */
+	if (psr_typ == 4 &&
+	   ((psr_vers == 0) ||
+	    ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
+		init_viking();
+		return;
+	}
+
+	/* Finally the Tsunami. */
+	if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
+		init_tsunami();
+		return;
+	}
+
+	/* Oh well */
+	srmmu_is_bad();
+}
+
+#ifdef CONFIG_SMP
+/* Local cross-calls. */
+static void smp_flush_page_for_dma(unsigned long page)
+{
+	xc1(local_ops->page_for_dma, page);
+	local_ops->page_for_dma(page);
+}
+
+static void smp_flush_cache_all(void)
+{
+	xc0(local_ops->cache_all);
+	local_ops->cache_all();
+}
+
+static void smp_flush_tlb_all(void)
+{
+	xc0(local_ops->tlb_all);
+	local_ops->tlb_all();
+}
+
+static bool any_other_mm_cpus(struct mm_struct *mm)
+{
+	return cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids;
+}
+
+static void smp_flush_cache_mm(struct mm_struct *mm)
+{
+	if (mm->context != NO_CONTEXT) {
+		if (any_other_mm_cpus(mm))
+			xc1(local_ops->cache_mm, (unsigned long)mm);
+		local_ops->cache_mm(mm);
+	}
+}
+
+static void smp_flush_tlb_mm(struct mm_struct *mm)
+{
+	if (mm->context != NO_CONTEXT) {
+		if (any_other_mm_cpus(mm)) {
+			xc1(local_ops->tlb_mm, (unsigned long)mm);
+			if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
+				cpumask_copy(mm_cpumask(mm),
+					     cpumask_of(smp_processor_id()));
+		}
+		local_ops->tlb_mm(mm);
+	}
+}
+
+static void smp_flush_cache_range(struct vm_area_struct *vma,
+				  unsigned long start,
+				  unsigned long end)
+{
+	struct mm_struct *mm = vma->vm_mm;
+
+	if (mm->context != NO_CONTEXT) {
+		if (any_other_mm_cpus(mm))
+			xc3(local_ops->cache_range, (unsigned long)vma, start,
+			    end);
+		local_ops->cache_range(vma, start, end);
+	}
+}
+
+static void smp_flush_tlb_range(struct vm_area_struct *vma,
+				unsigned long start,
+				unsigned long end)
+{
+	struct mm_struct *mm = vma->vm_mm;
+
+	if (mm->context != NO_CONTEXT) {
+		if (any_other_mm_cpus(mm))
+			xc3(local_ops->tlb_range, (unsigned long)vma, start,
+			    end);
+		local_ops->tlb_range(vma, start, end);
+	}
+}
+
+static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
+{
+	struct mm_struct *mm = vma->vm_mm;
+
+	if (mm->context != NO_CONTEXT) {
+		if (any_other_mm_cpus(mm))
+			xc2(local_ops->cache_page, (unsigned long)vma, page);
+		local_ops->cache_page(vma, page);
+	}
+}
+
+static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
+{
+	struct mm_struct *mm = vma->vm_mm;
+
+	if (mm->context != NO_CONTEXT) {
+		if (any_other_mm_cpus(mm))
+			xc2(local_ops->tlb_page, (unsigned long)vma, page);
+		local_ops->tlb_page(vma, page);
+	}
+}
+
+static void smp_flush_page_to_ram(unsigned long page)
+{
+	/* Current theory is that those who call this are the one's
+	 * who have just dirtied their cache with the pages contents
+	 * in kernel space, therefore we only run this on local cpu.
+	 *
+	 * XXX This experiment failed, research further... -DaveM
+	 */
+#if 1
+	xc1(local_ops->page_to_ram, page);
+#endif
+	local_ops->page_to_ram(page);
+}
+
+static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
+{
+	if (any_other_mm_cpus(mm))
+		xc2(local_ops->sig_insns, (unsigned long)mm, insn_addr);
+	local_ops->sig_insns(mm, insn_addr);
+}
+
+static struct sparc32_cachetlb_ops smp_cachetlb_ops __ro_after_init = {
+	.cache_all	= smp_flush_cache_all,
+	.cache_mm	= smp_flush_cache_mm,
+	.cache_page	= smp_flush_cache_page,
+	.cache_range	= smp_flush_cache_range,
+	.tlb_all	= smp_flush_tlb_all,
+	.tlb_mm		= smp_flush_tlb_mm,
+	.tlb_page	= smp_flush_tlb_page,
+	.tlb_range	= smp_flush_tlb_range,
+	.page_to_ram	= smp_flush_page_to_ram,
+	.sig_insns	= smp_flush_sig_insns,
+	.page_for_dma	= smp_flush_page_for_dma,
+};
+#endif
+
+/* Load up routines and constants for sun4m and sun4d mmu */
+void __init load_mmu(void)
+{
+	/* Functions */
+	get_srmmu_type();
+
+#ifdef CONFIG_SMP
+	/* El switcheroo... */
+	local_ops = sparc32_cachetlb_ops;
+
+	if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) {
+		smp_cachetlb_ops.tlb_all = local_ops->tlb_all;
+		smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm;
+		smp_cachetlb_ops.tlb_range = local_ops->tlb_range;
+		smp_cachetlb_ops.tlb_page = local_ops->tlb_page;
+	}
+
+	if (poke_srmmu == poke_viking) {
+		/* Avoid unnecessary cross calls. */
+		smp_cachetlb_ops.cache_all = local_ops->cache_all;
+		smp_cachetlb_ops.cache_mm = local_ops->cache_mm;
+		smp_cachetlb_ops.cache_range = local_ops->cache_range;
+		smp_cachetlb_ops.cache_page = local_ops->cache_page;
+
+		smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram;
+		smp_cachetlb_ops.sig_insns = local_ops->sig_insns;
+		smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma;
+	}
+
+	/* It really is const after this point. */
+	sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *)
+		&smp_cachetlb_ops;
+#endif
+
+	if (sparc_cpu_model != sun4d)
+		ld_mmu_iommu();
+#ifdef CONFIG_SMP
+	if (sparc_cpu_model == sun4d)
+		sun4d_init_smp();
+	else if (sparc_cpu_model == sparc_leon)
+		leon_init_smp();
+	else
+		sun4m_init_smp();
+#endif
+}
diff --git a/arch/sparc/mm/srmmu_access.S b/arch/sparc/mm/srmmu_access.S
new file mode 100644
index 000000000..d8d2e644a
--- /dev/null
+++ b/arch/sparc/mm/srmmu_access.S
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Assembler variants of srmmu access functions.
+ * Implemented in assembler to allow run-time patching.
+ * LEON uses a different ASI for MMUREGS than SUN.
+ *
+ * The leon_1insn_patch infrastructure is used
+ * for the run-time patching.
+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asmmacro.h>
+#include <asm/pgtsrmmu.h>
+#include <asm/asi.h>
+
+/* unsigned int srmmu_get_mmureg(void) */
+ENTRY(srmmu_get_mmureg)
+LEON_PI(lda	[%g0] ASI_LEON_MMUREGS, %o0)
+SUN_PI_(lda	[%g0] ASI_M_MMUREGS, %o0)
+	retl
+	 nop
+ENDPROC(srmmu_get_mmureg)
+
+/* void srmmu_set_mmureg(unsigned long regval) */
+ENTRY(srmmu_set_mmureg)
+LEON_PI(sta	%o0, [%g0] ASI_LEON_MMUREGS)
+SUN_PI_(sta	%o0, [%g0] ASI_M_MMUREGS)
+	retl
+	 nop
+ENDPROC(srmmu_set_mmureg)
+
+/* void srmmu_set_ctable_ptr(unsigned long paddr) */
+ENTRY(srmmu_set_ctable_ptr)
+	/* paddr = ((paddr >> 4) & SRMMU_CTX_PMASK); */
+	srl	%o0, 4, %g1
+	and	%g1, SRMMU_CTX_PMASK, %g1
+
+	mov	SRMMU_CTXTBL_PTR, %g2
+LEON_PI(sta	%g1, [%g2] ASI_LEON_MMUREGS)
+SUN_PI_(sta	%g1, [%g2] ASI_M_MMUREGS)
+	retl
+	 nop
+ENDPROC(srmmu_set_ctable_ptr)
+
+
+/* void srmmu_set_context(int context) */
+ENTRY(srmmu_set_context)
+	mov	SRMMU_CTX_REG, %g1
+LEON_PI(sta	%o0, [%g1] ASI_LEON_MMUREGS)
+SUN_PI_(sta	%o0, [%g1] ASI_M_MMUREGS)
+	retl
+	 nop
+ENDPROC(srmmu_set_context)
+
+
+/* int srmmu_get_context(void) */
+ENTRY(srmmu_get_context)
+	mov	SRMMU_CTX_REG, %o0
+LEON_PI(lda     [%o0] ASI_LEON_MMUREGS, %o0)
+SUN_PI_(lda	[%o0] ASI_M_MMUREGS, %o0)
+	retl
+	 nop
+ENDPROC(srmmu_get_context)
+
+
+/* unsigned int srmmu_get_fstatus(void) */
+ENTRY(srmmu_get_fstatus)
+	mov	SRMMU_FAULT_STATUS, %o0
+LEON_PI(lda     [%o0] ASI_LEON_MMUREGS, %o0)
+SUN_PI_(lda	[%o0] ASI_M_MMUREGS, %o0)
+	retl
+	 nop
+ENDPROC(srmmu_get_fstatus)
+
+
+/* unsigned int srmmu_get_faddr(void) */
+ENTRY(srmmu_get_faddr)
+	mov	SRMMU_FAULT_ADDR, %o0
+LEON_PI(lda     [%o0] ASI_LEON_MMUREGS, %o0)
+SUN_PI_(lda	[%o0] ASI_M_MMUREGS, %o0)
+	retl
+	 nop
+ENDPROC(srmmu_get_faddr)
diff --git a/arch/sparc/mm/swift.S b/arch/sparc/mm/swift.S
new file mode 100644
index 000000000..f414bfd8d
--- /dev/null
+++ b/arch/sparc/mm/swift.S
@@ -0,0 +1,256 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * swift.S: MicroSparc-II mmu/cache operations.
+ *
+ * Copyright (C) 1999 David S. Miller (davem@redhat.com)
+ */
+
+#include <asm/psr.h>
+#include <asm/asi.h>
+#include <asm/page.h>
+#include <asm/pgtsrmmu.h>
+#include <asm/asm-offsets.h>
+
+	.text
+	.align	4
+
+#if 1	/* XXX screw this, I can't get the VAC flushes working
+	 * XXX reliably... -DaveM
+	 */
+	.globl	swift_flush_cache_all, swift_flush_cache_mm
+	.globl	swift_flush_cache_range, swift_flush_cache_page
+	.globl	swift_flush_page_for_dma
+	.globl	swift_flush_page_to_ram
+
+swift_flush_cache_all:
+swift_flush_cache_mm:
+swift_flush_cache_range:
+swift_flush_cache_page:
+swift_flush_page_for_dma:
+swift_flush_page_to_ram:
+	sethi	%hi(0x2000), %o0
+1:	subcc	%o0, 0x10, %o0
+	add	%o0, %o0, %o1
+	sta	%g0, [%o0] ASI_M_DATAC_TAG
+	bne	1b
+	 sta	%g0, [%o1] ASI_M_TXTC_TAG
+	retl
+	 nop
+#else
+
+	.globl	swift_flush_cache_all
+swift_flush_cache_all:
+	WINDOW_FLUSH(%g4, %g5)
+
+	/* Just clear out all the tags. */
+	sethi	%hi(16 * 1024), %o0
+1:	subcc	%o0, 16, %o0
+	sta	%g0, [%o0] ASI_M_TXTC_TAG
+	bne	1b
+	 sta	%g0, [%o0] ASI_M_DATAC_TAG
+	retl
+	 nop
+
+	.globl	swift_flush_cache_mm
+swift_flush_cache_mm:
+	ld	[%o0 + AOFF_mm_context], %g2
+	cmp	%g2, -1
+	be	swift_flush_cache_mm_out
+	WINDOW_FLUSH(%g4, %g5)
+	rd	%psr, %g1
+	andn	%g1, PSR_ET, %g3
+	wr	%g3, 0x0, %psr
+	nop
+	nop
+	mov	SRMMU_CTX_REG, %g7
+	lda	[%g7] ASI_M_MMUREGS, %g5
+	sta	%g2, [%g7] ASI_M_MMUREGS
+
+#if 1
+	sethi	%hi(0x2000), %o0
+1:	subcc	%o0, 0x10, %o0
+	sta	%g0, [%o0] ASI_M_FLUSH_CTX
+	bne	1b
+	 nop
+#else
+	clr	%o0
+	or	%g0, 2048, %g7
+	or	%g0, 2048, %o1
+	add	%o1, 2048, %o2
+	add	%o2, 2048, %o3
+	mov	16, %o4
+	add	%o4, 2048, %o5
+	add	%o5, 2048, %g2
+	add	%g2, 2048, %g3
+1:	sta	%g0, [%o0      ] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %o1] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %o2] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %o3] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %o4] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %o5] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_CTX
+	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_CTX
+	subcc	%g7, 32, %g7
+	bne	1b
+	 add	%o0, 32, %o0
+#endif
+
+	mov	SRMMU_CTX_REG, %g7
+	sta	%g5, [%g7] ASI_M_MMUREGS
+	wr	%g1, 0x0, %psr
+	nop
+	nop
+swift_flush_cache_mm_out:
+	retl
+	 nop
+
+	.globl	swift_flush_cache_range
+swift_flush_cache_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+	sub	%o2, %o1, %o2
+	sethi	%hi(4096), %o3
+	cmp	%o2, %o3
+	bgu	swift_flush_cache_mm
+	 nop
+	b	70f
+	 nop
+
+	.globl	swift_flush_cache_page
+swift_flush_cache_page:
+	ld	[%o0 + VMA_VM_MM], %o0
+70:
+	ld	[%o0 + AOFF_mm_context], %g2
+	cmp	%g2, -1
+	be	swift_flush_cache_page_out
+	WINDOW_FLUSH(%g4, %g5)
+	rd	%psr, %g1
+	andn	%g1, PSR_ET, %g3
+	wr	%g3, 0x0, %psr
+	nop
+	nop
+	mov	SRMMU_CTX_REG, %g7
+	lda	[%g7] ASI_M_MMUREGS, %g5
+	sta	%g2, [%g7] ASI_M_MMUREGS
+
+	andn	%o1, (PAGE_SIZE - 1), %o1
+#if 1
+	sethi	%hi(0x1000), %o0
+1:	subcc	%o0, 0x10, %o0
+	sta	%g0, [%o1 + %o0] ASI_M_FLUSH_PAGE
+	bne	1b
+	 nop
+#else
+	or	%g0, 512, %g7
+	or	%g0, 512, %o0
+	add	%o0, 512, %o2
+	add	%o2, 512, %o3
+	add	%o3, 512, %o4
+	add	%o4, 512, %o5
+	add	%o5, 512, %g3
+	add	%g3, 512, %g4
+1:	sta	%g0, [%o1      ] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o0] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o2] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o3] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
+	subcc	%g7, 16, %g7
+	bne	1b
+	 add	%o1, 16, %o1
+#endif
+
+	mov	SRMMU_CTX_REG, %g7
+	sta	%g5, [%g7] ASI_M_MMUREGS
+	wr	%g1, 0x0, %psr
+	nop
+	nop
+swift_flush_cache_page_out:
+	retl
+	 nop
+
+	/* Swift is write-thru, however it is not
+	 * I/O nor TLB-walk coherent.  Also it has
+	 * caches which are virtually indexed and tagged.
+	 */
+	.globl	swift_flush_page_for_dma
+	.globl	swift_flush_page_to_ram
+swift_flush_page_for_dma:
+swift_flush_page_to_ram:
+	andn	%o0, (PAGE_SIZE - 1), %o1
+#if 1
+	sethi	%hi(0x1000), %o0
+1:	subcc	%o0, 0x10, %o0
+	sta	%g0, [%o1 + %o0] ASI_M_FLUSH_PAGE
+	bne	1b
+	 nop
+#else
+	or	%g0, 512, %g7
+	or	%g0, 512, %o0
+	add	%o0, 512, %o2
+	add	%o2, 512, %o3
+	add	%o3, 512, %o4
+	add	%o4, 512, %o5
+	add	%o5, 512, %g3
+	add	%g3, 512, %g4
+1:	sta	%g0, [%o1      ] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o0] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o2] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o3] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
+	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
+	subcc	%g7, 16, %g7
+	bne	1b
+	 add	%o1, 16, %o1
+#endif
+	retl
+	 nop
+#endif
+
+	.globl	swift_flush_sig_insns
+swift_flush_sig_insns:
+	flush	%o1
+	retl
+	 flush	%o1 + 4
+
+	.globl	swift_flush_tlb_mm
+	.globl	swift_flush_tlb_range
+	.globl	swift_flush_tlb_all
+swift_flush_tlb_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+swift_flush_tlb_mm:
+	ld	[%o0 + AOFF_mm_context], %g2
+	cmp	%g2, -1
+	be	swift_flush_tlb_all_out
+swift_flush_tlb_all:
+	mov	0x400, %o1
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+swift_flush_tlb_all_out:
+	retl
+	 nop
+
+	.globl	swift_flush_tlb_page
+swift_flush_tlb_page:
+	ld	[%o0 + VMA_VM_MM], %o0
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o3
+	andn	%o1, (PAGE_SIZE - 1), %o1
+	cmp	%o3, -1
+	be	swift_flush_tlb_page_out
+	 nop
+#if 1
+	mov	0x400, %o1
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE	
+#else
+	lda	[%g1] ASI_M_MMUREGS, %g5
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%o1] ASI_M_FLUSH_PAGE	/* rem. virt. cache. prot. */
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	sta	%g5, [%g1] ASI_M_MMUREGS
+#endif
+swift_flush_tlb_page_out:
+	retl
+	 nop
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
new file mode 100644
index 000000000..6d9dd5eb1
--- /dev/null
+++ b/arch/sparc/mm/tlb.c
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0
+/* arch/sparc64/mm/tlb.c
+ *
+ * Copyright (C) 2004 David S. Miller <davem@redhat.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/percpu.h>
+#include <linux/mm.h>
+#include <linux/swap.h>
+#include <linux/preempt.h>
+#include <linux/pagemap.h>
+
+#include <kunit/visibility.h>
+
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+#include <asm/mmu_context.h>
+#include <asm/tlb.h>
+
+/* Heavily inspired by the ppc64 code.  */
+
+static DEFINE_PER_CPU(struct tlb_batch, tlb_batch);
+
+void flush_tlb_pending(void)
+{
+	struct tlb_batch *tb = &get_cpu_var(tlb_batch);
+	struct mm_struct *mm = tb->mm;
+
+	if (!tb->tlb_nr)
+		goto out;
+
+	flush_tsb_user(tb);
+
+	if (CTX_VALID(mm->context)) {
+		if (tb->tlb_nr == 1) {
+			global_flush_tlb_page(mm, tb->vaddrs[0]);
+		} else {
+#ifdef CONFIG_SMP
+			smp_flush_tlb_pending(tb->mm, tb->tlb_nr,
+					      &tb->vaddrs[0]);
+#else
+			__flush_tlb_pending(CTX_HWBITS(tb->mm->context),
+					    tb->tlb_nr, &tb->vaddrs[0]);
+#endif
+		}
+	}
+
+	tb->tlb_nr = 0;
+
+out:
+	put_cpu_var(tlb_batch);
+}
+
+void arch_enter_lazy_mmu_mode(void)
+{
+	preempt_disable();
+}
+/* For lazy_mmu_mode KUnit tests */
+EXPORT_SYMBOL_IF_KUNIT(arch_enter_lazy_mmu_mode);
+
+void arch_flush_lazy_mmu_mode(void)
+{
+	struct tlb_batch *tb = this_cpu_ptr(&tlb_batch);
+
+	if (tb->tlb_nr)
+		flush_tlb_pending();
+}
+EXPORT_SYMBOL_IF_KUNIT(arch_flush_lazy_mmu_mode);
+
+void arch_leave_lazy_mmu_mode(void)
+{
+	arch_flush_lazy_mmu_mode();
+	preempt_enable();
+}
+EXPORT_SYMBOL_IF_KUNIT(arch_leave_lazy_mmu_mode);
+
+static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
+			      bool exec, unsigned int hugepage_shift)
+{
+	struct tlb_batch *tb = &get_cpu_var(tlb_batch);
+	unsigned long nr;
+
+	vaddr &= PAGE_MASK;
+	if (exec)
+		vaddr |= 0x1UL;
+
+	nr = tb->tlb_nr;
+
+	if (unlikely(nr != 0 && mm != tb->mm)) {
+		flush_tlb_pending();
+		nr = 0;
+	}
+
+	if (!is_lazy_mmu_mode_active()) {
+		flush_tsb_user_page(mm, vaddr, hugepage_shift);
+		global_flush_tlb_page(mm, vaddr);
+		goto out;
+	}
+
+	if (nr == 0) {
+		tb->mm = mm;
+		tb->hugepage_shift = hugepage_shift;
+	}
+
+	if (tb->hugepage_shift != hugepage_shift) {
+		flush_tlb_pending();
+		tb->hugepage_shift = hugepage_shift;
+		nr = 0;
+	}
+
+	tb->vaddrs[nr] = vaddr;
+	tb->tlb_nr = ++nr;
+	if (nr >= TLB_BATCH_NR)
+		flush_tlb_pending();
+
+out:
+	put_cpu_var(tlb_batch);
+}
+
+void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
+		   pte_t *ptep, pte_t orig, int fullmm,
+		   unsigned int hugepage_shift)
+{
+	if (tlb_type != hypervisor &&
+	    pte_dirty(orig)) {
+		unsigned long paddr, pfn = pte_pfn(orig);
+		struct address_space *mapping;
+		struct page *page;
+		struct folio *folio;
+
+		if (!pfn_valid(pfn))
+			goto no_cache_flush;
+
+		page = pfn_to_page(pfn);
+		if (PageReserved(page))
+			goto no_cache_flush;
+
+		/* A real file page? */
+		folio = page_folio(page);
+		mapping = folio_flush_mapping(folio);
+		if (!mapping)
+			goto no_cache_flush;
+
+		paddr = (unsigned long) page_address(page);
+		if ((paddr ^ vaddr) & (1 << 13))
+			flush_dcache_folio_all(mm, folio);
+	}
+
+no_cache_flush:
+	if (!fullmm)
+		tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift);
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
+			       pmd_t pmd)
+{
+	unsigned long end;
+	pte_t *pte;
+
+	pte = pte_offset_map(&pmd, vaddr);
+	if (!pte)
+		return;
+	end = vaddr + HPAGE_SIZE;
+	while (vaddr < end) {
+		if (pte_val(*pte) & _PAGE_VALID) {
+			bool exec = pte_exec(*pte);
+
+			tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT);
+		}
+		pte++;
+		vaddr += PAGE_SIZE;
+	}
+	pte_unmap(pte);
+}
+
+
+static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr,
+			   pmd_t orig, pmd_t pmd)
+{
+	if (mm == &init_mm)
+		return;
+
+	if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
+		/*
+		 * Note that this routine only sets pmds for THP pages.
+		 * Hugetlb pages are handled elsewhere.  We need to check
+		 * for huge zero page.  Huge zero pages are like hugetlb
+		 * pages in that there is no RSS, but there is the need
+		 * for TSB entries.  So, huge zero page counts go into
+		 * hugetlb_pte_count.
+		 */
+		if (pmd_val(pmd) & _PAGE_PMD_HUGE) {
+			if (is_huge_zero_pmd(pmd))
+				mm->context.hugetlb_pte_count++;
+			else
+				mm->context.thp_pte_count++;
+		} else {
+			if (is_huge_zero_pmd(orig))
+				mm->context.hugetlb_pte_count--;
+			else
+				mm->context.thp_pte_count--;
+		}
+
+		/* Do not try to allocate the TSB hash table if we
+		 * don't have one already.  We have various locks held
+		 * and thus we'll end up doing a GFP_KERNEL allocation
+		 * in an atomic context.
+		 *
+		 * Instead, we let the first TLB miss on a hugepage
+		 * take care of this.
+		 */
+	}
+
+	if (!pmd_none(orig)) {
+		addr &= HPAGE_MASK;
+		if (pmd_trans_huge(orig)) {
+			pte_t orig_pte = __pte(pmd_val(orig));
+			bool exec = pte_exec(orig_pte);
+
+			tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);
+			tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
+					  REAL_HPAGE_SHIFT);
+		} else {
+			tlb_batch_pmd_scan(mm, addr, orig);
+		}
+	}
+}
+
+void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+		pmd_t *pmdp, pmd_t pmd)
+{
+	pmd_t orig = *pmdp;
+
+	*pmdp = pmd;
+	__set_pmd_acct(mm, addr, orig, pmd);
+}
+
+static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
+		unsigned long address, pmd_t *pmdp, pmd_t pmd)
+{
+	pmd_t old;
+
+	do {
+		old = *pmdp;
+	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
+	__set_pmd_acct(vma->vm_mm, address, old, pmd);
+
+	return old;
+}
+
+/*
+ * This routine is only called when splitting a THP
+ */
+pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
+		     pmd_t *pmdp)
+{
+	pmd_t old, entry;
+
+	VM_WARN_ON_ONCE(!pmd_present(*pmdp));
+	entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID);
+	old = pmdp_establish(vma, address, pmdp, entry);
+	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+
+	/*
+	 * set_pmd_at() will not be called in a way to decrement
+	 * thp_pte_count when splitting a THP, so do it now.
+	 * Sanity check pmd before doing the actual decrement.
+	 */
+	if ((pmd_val(entry) & _PAGE_PMD_HUGE) &&
+	    !is_huge_zero_pmd(entry))
+		(vma->vm_mm)->context.thp_pte_count--;
+
+	return old;
+}
+
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
+				pgtable_t pgtable)
+{
+	struct list_head *lh = (struct list_head *) pgtable;
+
+	assert_spin_locked(&mm->page_table_lock);
+
+	/* FIFO */
+	if (!pmd_huge_pte(mm, pmdp))
+		INIT_LIST_HEAD(lh);
+	else
+		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
+	pmd_huge_pte(mm, pmdp) = pgtable;
+}
+
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
+{
+	struct list_head *lh;
+	pgtable_t pgtable;
+
+	assert_spin_locked(&mm->page_table_lock);
+
+	/* FIFO */
+	pgtable = pmd_huge_pte(mm, pmdp);
+	lh = (struct list_head *) pgtable;
+	if (list_empty(lh))
+		pmd_huge_pte(mm, pmdp) = NULL;
+	else {
+		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
+		list_del(lh);
+	}
+	pte_val(pgtable[0]) = 0;
+	pte_val(pgtable[1]) = 0;
+
+	return pgtable;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
new file mode 100644
index 000000000..5fe52a64c
--- /dev/null
+++ b/arch/sparc/mm/tsb.c
@@ -0,0 +1,635 @@
+// SPDX-License-Identifier: GPL-2.0
+/* arch/sparc64/mm/tsb.c
+ *
+ * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net>
+ */
+
+#include <linux/kernel.h>
+#include <linux/preempt.h>
+#include <linux/slab.h>
+#include <linux/mm_types.h>
+#include <linux/pgtable.h>
+
+#include <asm/page.h>
+#include <asm/mmu_context.h>
+#include <asm/setup.h>
+#include <asm/tsb.h>
+#include <asm/tlb.h>
+#include <asm/oplib.h>
+
+extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
+
+static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries)
+{
+	vaddr >>= hash_shift;
+	return vaddr & (nentries - 1);
+}
+
+static inline int tag_compare(unsigned long tag, unsigned long vaddr)
+{
+	return (tag == (vaddr >> 22));
+}
+
+static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
+{
+	unsigned long idx;
+
+	for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
+		struct tsb *ent = &swapper_tsb[idx];
+		unsigned long match = idx << 13;
+
+		match |= (ent->tag << 22);
+		if (match >= start && match < end)
+			ent->tag = (1UL << TSB_TAG_INVALID_BIT);
+	}
+}
+
+/* TSB flushes need only occur on the processor initiating the address
+ * space modification, not on each cpu the address space has run on.
+ * Only the TLB flush needs that treatment.
+ */
+
+void flush_tsb_kernel_range(unsigned long start, unsigned long end)
+{
+	unsigned long v;
+
+	if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
+		return flush_tsb_kernel_range_scan(start, end);
+
+	for (v = start; v < end; v += PAGE_SIZE) {
+		unsigned long hash = tsb_hash(v, PAGE_SHIFT,
+					      KERNEL_TSB_NENTRIES);
+		struct tsb *ent = &swapper_tsb[hash];
+
+		if (tag_compare(ent->tag, v))
+			ent->tag = (1UL << TSB_TAG_INVALID_BIT);
+	}
+}
+
+static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v,
+				  unsigned long hash_shift,
+				  unsigned long nentries)
+{
+	unsigned long tag, ent, hash;
+
+	v &= ~0x1UL;
+	hash = tsb_hash(v, hash_shift, nentries);
+	ent = tsb + (hash * sizeof(struct tsb));
+	tag = (v >> 22UL);
+
+	tsb_flush(ent, tag);
+}
+
+static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift,
+			    unsigned long tsb, unsigned long nentries)
+{
+	unsigned long i;
+
+	for (i = 0; i < tb->tlb_nr; i++)
+		__flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries);
+}
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+static void __flush_huge_tsb_one_entry(unsigned long tsb, unsigned long v,
+				       unsigned long hash_shift,
+				       unsigned long nentries,
+				       unsigned int hugepage_shift)
+{
+	unsigned int hpage_entries;
+	unsigned int i;
+
+	hpage_entries = 1 << (hugepage_shift - hash_shift);
+	for (i = 0; i < hpage_entries; i++)
+		__flush_tsb_one_entry(tsb, v + (i << hash_shift), hash_shift,
+				      nentries);
+}
+
+static void __flush_huge_tsb_one(struct tlb_batch *tb, unsigned long hash_shift,
+				 unsigned long tsb, unsigned long nentries,
+				 unsigned int hugepage_shift)
+{
+	unsigned long i;
+
+	for (i = 0; i < tb->tlb_nr; i++)
+		__flush_huge_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift,
+					   nentries, hugepage_shift);
+}
+#endif
+
+void flush_tsb_user(struct tlb_batch *tb)
+{
+	struct mm_struct *mm = tb->mm;
+	unsigned long nentries, base, flags;
+
+	spin_lock_irqsave(&mm->context.lock, flags);
+
+	if (tb->hugepage_shift < REAL_HPAGE_SHIFT) {
+		base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+		nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+		if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+			base = __pa(base);
+		if (tb->hugepage_shift == PAGE_SHIFT)
+			__flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
+#if defined(CONFIG_HUGETLB_PAGE)
+		else
+			__flush_huge_tsb_one(tb, PAGE_SHIFT, base, nentries,
+					     tb->hugepage_shift);
+#endif
+	}
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+		base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
+		nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
+		if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+			base = __pa(base);
+		__flush_huge_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries,
+				     tb->hugepage_shift);
+	}
+#endif
+	spin_unlock_irqrestore(&mm->context.lock, flags);
+}
+
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr,
+			 unsigned int hugepage_shift)
+{
+	unsigned long nentries, base, flags;
+
+	spin_lock_irqsave(&mm->context.lock, flags);
+
+	if (hugepage_shift < REAL_HPAGE_SHIFT) {
+		base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+		nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+		if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+			base = __pa(base);
+		if (hugepage_shift == PAGE_SHIFT)
+			__flush_tsb_one_entry(base, vaddr, PAGE_SHIFT,
+					      nentries);
+#if defined(CONFIG_HUGETLB_PAGE)
+		else
+			__flush_huge_tsb_one_entry(base, vaddr, PAGE_SHIFT,
+						   nentries, hugepage_shift);
+#endif
+	}
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+		base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
+		nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
+		if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+			base = __pa(base);
+		__flush_huge_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT,
+					   nentries, hugepage_shift);
+	}
+#endif
+	spin_unlock_irqrestore(&mm->context.lock, flags);
+}
+
+#define HV_PGSZ_IDX_BASE	HV_PGSZ_IDX_8K
+#define HV_PGSZ_MASK_BASE	HV_PGSZ_MASK_8K
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+#define HV_PGSZ_IDX_HUGE	HV_PGSZ_IDX_4MB
+#define HV_PGSZ_MASK_HUGE	HV_PGSZ_MASK_4MB
+#endif
+
+static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes)
+{
+	unsigned long tsb_reg, base, tsb_paddr;
+	unsigned long page_sz, tte;
+
+	mm->context.tsb_block[tsb_idx].tsb_nentries =
+		tsb_bytes / sizeof(struct tsb);
+
+	switch (tsb_idx) {
+	case MM_TSB_BASE:
+		base = TSBMAP_8K_BASE;
+		break;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	case MM_TSB_HUGE:
+		base = TSBMAP_4M_BASE;
+		break;
+#endif
+	default:
+		BUG();
+	}
+
+	tte = pgprot_val(PAGE_KERNEL_LOCKED);
+	tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb);
+	BUG_ON(tsb_paddr & (tsb_bytes - 1UL));
+
+	/* Use the smallest page size that can map the whole TSB
+	 * in one TLB entry.
+	 */
+	switch (tsb_bytes) {
+	case 8192 << 0:
+		tsb_reg = 0x0UL;
+#ifdef DCACHE_ALIASING_POSSIBLE
+		base += (tsb_paddr & 8192);
+#endif
+		page_sz = 8192;
+		break;
+
+	case 8192 << 1:
+		tsb_reg = 0x1UL;
+		page_sz = 64 * 1024;
+		break;
+
+	case 8192 << 2:
+		tsb_reg = 0x2UL;
+		page_sz = 64 * 1024;
+		break;
+
+	case 8192 << 3:
+		tsb_reg = 0x3UL;
+		page_sz = 64 * 1024;
+		break;
+
+	case 8192 << 4:
+		tsb_reg = 0x4UL;
+		page_sz = 512 * 1024;
+		break;
+
+	case 8192 << 5:
+		tsb_reg = 0x5UL;
+		page_sz = 512 * 1024;
+		break;
+
+	case 8192 << 6:
+		tsb_reg = 0x6UL;
+		page_sz = 512 * 1024;
+		break;
+
+	case 8192 << 7:
+		tsb_reg = 0x7UL;
+		page_sz = 4 * 1024 * 1024;
+		break;
+
+	default:
+		printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n",
+		       current->comm, current->pid, tsb_bytes);
+		BUG();
+	}
+	tte |= pte_sz_bits(page_sz);
+
+	if (tlb_type == cheetah_plus || tlb_type == hypervisor) {
+		/* Physical mapping, no locked TLB entry for TSB.  */
+		tsb_reg |= tsb_paddr;
+
+		mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
+		mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0;
+		mm->context.tsb_block[tsb_idx].tsb_map_pte = 0;
+	} else {
+		tsb_reg |= base;
+		tsb_reg |= (tsb_paddr & (page_sz - 1UL));
+		tte |= (tsb_paddr & ~(page_sz - 1UL));
+
+		mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
+		mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base;
+		mm->context.tsb_block[tsb_idx].tsb_map_pte = tte;
+	}
+
+	/* Setup the Hypervisor TSB descriptor.  */
+	if (tlb_type == hypervisor) {
+		struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx];
+
+		switch (tsb_idx) {
+		case MM_TSB_BASE:
+			hp->pgsz_idx = HV_PGSZ_IDX_BASE;
+			break;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+		case MM_TSB_HUGE:
+			hp->pgsz_idx = HV_PGSZ_IDX_HUGE;
+			break;
+#endif
+		default:
+			BUG();
+		}
+		hp->assoc = 1;
+		hp->num_ttes = tsb_bytes / 16;
+		hp->ctx_idx = 0;
+		switch (tsb_idx) {
+		case MM_TSB_BASE:
+			hp->pgsz_mask = HV_PGSZ_MASK_BASE;
+			break;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+		case MM_TSB_HUGE:
+			hp->pgsz_mask = HV_PGSZ_MASK_HUGE;
+			break;
+#endif
+		default:
+			BUG();
+		}
+		hp->tsb_base = tsb_paddr;
+		hp->resv = 0;
+	}
+}
+
+struct kmem_cache *pgtable_cache __read_mostly;
+
+static struct kmem_cache *tsb_caches[8] __read_mostly;
+
+static const char *tsb_cache_names[8] = {
+	"tsb_8KB",
+	"tsb_16KB",
+	"tsb_32KB",
+	"tsb_64KB",
+	"tsb_128KB",
+	"tsb_256KB",
+	"tsb_512KB",
+	"tsb_1MB",
+};
+
+void __init pgtable_cache_init(void)
+{
+	unsigned long i;
+
+	pgtable_cache = kmem_cache_create("pgtable_cache",
+					  PAGE_SIZE, PAGE_SIZE,
+					  0,
+					  _clear_page);
+	if (!pgtable_cache) {
+		prom_printf("pgtable_cache_init(): Could not create!\n");
+		prom_halt();
+	}
+
+	for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) {
+		unsigned long size = 8192 << i;
+		const char *name = tsb_cache_names[i];
+
+		tsb_caches[i] = kmem_cache_create(name,
+						  size, size,
+						  0, NULL);
+		if (!tsb_caches[i]) {
+			prom_printf("Could not create %s cache\n", name);
+			prom_halt();
+		}
+	}
+}
+
+int sysctl_tsb_ratio = -2;
+
+static unsigned long tsb_size_to_rss_limit(unsigned long new_size)
+{
+	unsigned long num_ents = (new_size / sizeof(struct tsb));
+
+	if (sysctl_tsb_ratio < 0)
+		return num_ents - (num_ents >> -sysctl_tsb_ratio);
+	else
+		return num_ents + (num_ents >> sysctl_tsb_ratio);
+}
+
+/* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
+ * do_sparc64_fault() invokes this routine to try and grow it.
+ *
+ * When we reach the maximum TSB size supported, we stick ~0UL into
+ * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
+ * will not trigger any longer.
+ *
+ * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
+ * of two.  The TSB must be aligned to its size, so f.e. a 512K TSB
+ * must be 512K aligned.  It also must be physically contiguous, so we
+ * cannot use vmalloc().
+ *
+ * The idea here is to grow the TSB when the RSS of the process approaches
+ * the number of entries that the current TSB can hold at once.  Currently,
+ * we trigger when the RSS hits 3/4 of the TSB capacity.
+ */
+void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss)
+{
+	unsigned long max_tsb_size = 1 * 1024 * 1024;
+	unsigned long new_size, old_size, flags;
+	struct tsb *old_tsb, *new_tsb;
+	unsigned long new_cache_index, old_cache_index;
+	unsigned long new_rss_limit;
+	gfp_t gfp_flags;
+
+	if (max_tsb_size > PAGE_SIZE << MAX_PAGE_ORDER)
+		max_tsb_size = PAGE_SIZE << MAX_PAGE_ORDER;
+
+	new_cache_index = 0;
+	for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) {
+		new_rss_limit = tsb_size_to_rss_limit(new_size);
+		if (new_rss_limit > rss)
+			break;
+		new_cache_index++;
+	}
+
+	if (new_size == max_tsb_size)
+		new_rss_limit = ~0UL;
+
+retry_tsb_alloc:
+	gfp_flags = GFP_KERNEL;
+	if (new_size > (PAGE_SIZE * 2))
+		gfp_flags |= __GFP_NOWARN | __GFP_NORETRY;
+
+	new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index],
+					gfp_flags, numa_node_id());
+	if (unlikely(!new_tsb)) {
+		/* Not being able to fork due to a high-order TSB
+		 * allocation failure is very bad behavior.  Just back
+		 * down to a 0-order allocation and force no TSB
+		 * growing for this address space.
+		 */
+		if (mm->context.tsb_block[tsb_index].tsb == NULL &&
+		    new_cache_index > 0) {
+			new_cache_index = 0;
+			new_size = 8192;
+			new_rss_limit = ~0UL;
+			goto retry_tsb_alloc;
+		}
+
+		/* If we failed on a TSB grow, we are under serious
+		 * memory pressure so don't try to grow any more.
+		 */
+		if (mm->context.tsb_block[tsb_index].tsb != NULL)
+			mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL;
+		return;
+	}
+
+	/* Mark all tags as invalid.  */
+	tsb_init(new_tsb, new_size);
+
+	/* Ok, we are about to commit the changes.  If we are
+	 * growing an existing TSB the locking is very tricky,
+	 * so WATCH OUT!
+	 *
+	 * We have to hold mm->context.lock while committing to the
+	 * new TSB, this synchronizes us with processors in
+	 * flush_tsb_user() and switch_mm() for this address space.
+	 *
+	 * But even with that lock held, processors run asynchronously
+	 * accessing the old TSB via TLB miss handling.  This is OK
+	 * because those actions are just propagating state from the
+	 * Linux page tables into the TSB, page table mappings are not
+	 * being changed.  If a real fault occurs, the processor will
+	 * synchronize with us when it hits flush_tsb_user(), this is
+	 * also true for the case where vmscan is modifying the page
+	 * tables.  The only thing we need to be careful with is to
+	 * skip any locked TSB entries during copy_tsb().
+	 *
+	 * When we finish committing to the new TSB, we have to drop
+	 * the lock and ask all other cpus running this address space
+	 * to run tsb_context_switch() to see the new TSB table.
+	 */
+	spin_lock_irqsave(&mm->context.lock, flags);
+
+	old_tsb = mm->context.tsb_block[tsb_index].tsb;
+	old_cache_index =
+		(mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL);
+	old_size = (mm->context.tsb_block[tsb_index].tsb_nentries *
+		    sizeof(struct tsb));
+
+
+	/* Handle multiple threads trying to grow the TSB at the same time.
+	 * One will get in here first, and bump the size and the RSS limit.
+	 * The others will get in here next and hit this check.
+	 */
+	if (unlikely(old_tsb &&
+		     (rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) {
+		spin_unlock_irqrestore(&mm->context.lock, flags);
+
+		kmem_cache_free(tsb_caches[new_cache_index], new_tsb);
+		return;
+	}
+
+	mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit;
+
+	if (old_tsb) {
+		extern void copy_tsb(unsigned long old_tsb_base,
+				     unsigned long old_tsb_size,
+				     unsigned long new_tsb_base,
+				     unsigned long new_tsb_size,
+				     unsigned long page_size_shift);
+		unsigned long old_tsb_base = (unsigned long) old_tsb;
+		unsigned long new_tsb_base = (unsigned long) new_tsb;
+
+		if (tlb_type == cheetah_plus || tlb_type == hypervisor) {
+			old_tsb_base = __pa(old_tsb_base);
+			new_tsb_base = __pa(new_tsb_base);
+		}
+		copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size,
+			tsb_index == MM_TSB_BASE ?
+			PAGE_SHIFT : REAL_HPAGE_SHIFT);
+	}
+
+	mm->context.tsb_block[tsb_index].tsb = new_tsb;
+	setup_tsb_params(mm, tsb_index, new_size);
+
+	spin_unlock_irqrestore(&mm->context.lock, flags);
+
+	/* If old_tsb is NULL, we're being invoked for the first time
+	 * from init_new_context().
+	 */
+	if (old_tsb) {
+		/* Reload it on the local cpu.  */
+		tsb_context_switch(mm);
+
+		/* Now force other processors to do the same.  */
+		preempt_disable();
+		smp_tsb_sync(mm);
+		preempt_enable();
+
+		/* Now it is safe to free the old tsb.  */
+		kmem_cache_free(tsb_caches[old_cache_index], old_tsb);
+	}
+}
+
+int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+{
+	unsigned long mm_rss = get_mm_rss(mm);
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	unsigned long saved_hugetlb_pte_count;
+	unsigned long saved_thp_pte_count;
+#endif
+	unsigned int i;
+
+	spin_lock_init(&mm->context.lock);
+
+	mm->context.sparc64_ctx_val = 0UL;
+
+	mm->context.tag_store = NULL;
+	spin_lock_init(&mm->context.tag_lock);
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	/* We reset them to zero because the fork() page copying
+	 * will re-increment the counters as the parent PTEs are
+	 * copied into the child address space.
+	 */
+	saved_hugetlb_pte_count = mm->context.hugetlb_pte_count;
+	saved_thp_pte_count = mm->context.thp_pte_count;
+	mm->context.hugetlb_pte_count = 0;
+	mm->context.thp_pte_count = 0;
+
+	mm_rss -= saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE);
+#endif
+
+	/* copy_mm() copies over the parent's mm_struct before calling
+	 * us, so we need to zero out the TSB pointer or else tsb_grow()
+	 * will be confused and think there is an older TSB to free up.
+	 */
+	for (i = 0; i < MM_NUM_TSBS; i++)
+		mm->context.tsb_block[i].tsb = NULL;
+
+	/* If this is fork, inherit the parent's TSB size.  We would
+	 * grow it to that size on the first page fault anyways.
+	 */
+	tsb_grow(mm, MM_TSB_BASE, mm_rss);
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+	if (unlikely(saved_hugetlb_pte_count + saved_thp_pte_count))
+		tsb_grow(mm, MM_TSB_HUGE,
+			 (saved_hugetlb_pte_count + saved_thp_pte_count) *
+			 REAL_HPAGE_PER_HPAGE);
+#endif
+
+	if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void tsb_destroy_one(struct tsb_config *tp)
+{
+	unsigned long cache_index;
+
+	if (!tp->tsb)
+		return;
+	cache_index = tp->tsb_reg_val & 0x7UL;
+	kmem_cache_free(tsb_caches[cache_index], tp->tsb);
+	tp->tsb = NULL;
+	tp->tsb_reg_val = 0UL;
+}
+
+void destroy_context(struct mm_struct *mm)
+{
+	unsigned long flags, i;
+
+	for (i = 0; i < MM_NUM_TSBS; i++)
+		tsb_destroy_one(&mm->context.tsb_block[i]);
+
+	spin_lock_irqsave(&ctx_alloc_lock, flags);
+
+	if (CTX_VALID(mm->context)) {
+		unsigned long nr = CTX_NRBITS(mm->context);
+		mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63));
+	}
+
+	spin_unlock_irqrestore(&ctx_alloc_lock, flags);
+
+	/* If ADI tag storage was allocated for this task, free it */
+	if (mm->context.tag_store) {
+		tag_storage_desc_t *tag_desc;
+		unsigned long max_desc;
+		unsigned char *tags;
+
+		tag_desc = mm->context.tag_store;
+		max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t);
+		for (i = 0; i < max_desc; i++) {
+			tags = tag_desc->tags;
+			tag_desc->tags = NULL;
+			kfree(tags);
+			tag_desc++;
+		}
+		kfree(mm->context.tag_store);
+		mm->context.tag_store = NULL;
+	}
+}
diff --git a/arch/sparc/mm/tsunami.S b/arch/sparc/mm/tsunami.S
new file mode 100644
index 000000000..62b742df6
--- /dev/null
+++ b/arch/sparc/mm/tsunami.S
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * tsunami.S: High speed MicroSparc-I mmu/cache operations.
+ *
+ * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
+ */
+
+#include <asm/ptrace.h>
+#include <asm/asm-offsets.h>
+#include <asm/psr.h>
+#include <asm/asi.h>
+#include <asm/page.h>
+#include <asm/pgtsrmmu.h>
+
+	.text
+	.align	4
+
+	.globl	tsunami_flush_cache_all, tsunami_flush_cache_mm
+	.globl	tsunami_flush_cache_range, tsunami_flush_cache_page
+	.globl	tsunami_flush_page_to_ram, tsunami_flush_page_for_dma
+	.globl	tsunami_flush_sig_insns
+	.globl	tsunami_flush_tlb_all, tsunami_flush_tlb_mm
+	.globl	tsunami_flush_tlb_range, tsunami_flush_tlb_page
+
+	/* Sliiick... */
+tsunami_flush_cache_page:
+tsunami_flush_cache_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+tsunami_flush_cache_mm:
+	ld	[%o0 + AOFF_mm_context], %g2
+	cmp	%g2, -1
+	be	tsunami_flush_cache_out
+tsunami_flush_cache_all:
+	WINDOW_FLUSH(%g4, %g5)
+tsunami_flush_page_for_dma:
+	sta	%g0, [%g0] ASI_M_IC_FLCLEAR
+	sta	%g0, [%g0] ASI_M_DC_FLCLEAR
+tsunami_flush_cache_out:
+tsunami_flush_page_to_ram:
+	retl
+	 nop
+
+tsunami_flush_sig_insns:
+	flush	%o1
+	retl
+	 flush	%o1 + 4
+
+	/* More slick stuff... */
+tsunami_flush_tlb_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+tsunami_flush_tlb_mm:
+	ld	[%o0 + AOFF_mm_context], %g2
+	cmp	%g2, -1
+	be	tsunami_flush_tlb_out
+tsunami_flush_tlb_all:
+	 mov	0x400, %o1
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	nop
+	nop
+	nop
+	nop
+	nop
+tsunami_flush_tlb_out:
+	retl
+	 nop
+
+	/* This one can be done in a fine grained manner... */
+tsunami_flush_tlb_page:
+	ld	[%o0 + VMA_VM_MM], %o0
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o3
+	andn	%o1, (PAGE_SIZE - 1), %o1
+	cmp	%o3, -1
+	be	tsunami_flush_tlb_page_out
+	 lda	[%g1] ASI_M_MMUREGS, %g5
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	nop
+	nop
+	nop
+	nop
+	nop
+tsunami_flush_tlb_page_out:
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+
+#define MIRROR_BLOCK(dst, src, offset, t0, t1, t2, t3) \
+	ldd	[src + offset + 0x18], t0; \
+	std	t0, [dst + offset + 0x18]; \
+	ldd	[src + offset + 0x10], t2; \
+	std	t2, [dst + offset + 0x10]; \
+	ldd	[src + offset + 0x08], t0; \
+	std	t0, [dst + offset + 0x08]; \
+	ldd	[src + offset + 0x00], t2; \
+	std	t2, [dst + offset + 0x00];
+
+tsunami_copy_1page:
+/* NOTE: This routine has to be shorter than 70insns --jj */
+	or	%g0, (PAGE_SIZE >> 8), %g1
+1:
+	MIRROR_BLOCK(%o0, %o1, 0x00, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0x20, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0x40, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0x60, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0x80, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0xa0, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0xc0, %o2, %o3, %o4, %o5)
+	MIRROR_BLOCK(%o0, %o1, 0xe0, %o2, %o3, %o4, %o5)
+	subcc	%g1, 1, %g1
+	add	%o0, 0x100, %o0
+	bne	1b
+	 add	%o1, 0x100, %o1
+
+	.globl	tsunami_setup_blockops
+tsunami_setup_blockops:
+	sethi	%hi(__copy_1page), %o0
+	or	%o0, %lo(__copy_1page), %o0
+	sethi	%hi(tsunami_copy_1page), %o1
+	or	%o1, %lo(tsunami_copy_1page), %o1
+	sethi	%hi(tsunami_setup_blockops), %o2
+	or	%o2, %lo(tsunami_setup_blockops), %o2
+	ld	[%o1], %o4
+1:	add	%o1, 4, %o1
+	st	%o4, [%o0]
+	add	%o0, 4, %o0
+	cmp	%o1, %o2
+	bne	1b
+	ld	[%o1], %o4
+	sta	%g0, [%g0] ASI_M_IC_FLCLEAR
+	sta	%g0, [%g0] ASI_M_DC_FLCLEAR
+	retl
+	 nop
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
new file mode 100644
index 000000000..70e658d10
--- /dev/null
+++ b/arch/sparc/mm/ultra.S
@@ -0,0 +1,1102 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ultra.S: Don't expand these all over the place...
+ *
+ * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
+ */
+
+#include <linux/pgtable.h>
+#include <asm/asi.h>
+#include <asm/page.h>
+#include <asm/spitfire.h>
+#include <asm/mmu_context.h>
+#include <asm/mmu.h>
+#include <asm/pil.h>
+#include <asm/head.h>
+#include <asm/thread_info.h>
+#include <asm/cacheflush.h>
+#include <asm/hypervisor.h>
+#include <asm/cpudata.h>
+
+	/* Basically, most of the Spitfire vs. Cheetah madness
+	 * has to do with the fact that Cheetah does not support
+	 * IMMU flushes out of the secondary context.  Someone needs
+	 * to throw a south lake birthday party for the folks
+	 * in Microelectronics who refused to fix this shit.
+	 */
+
+	/* This file is meant to be read efficiently by the CPU, not humans.
+	 * Staraj sie tego nikomu nie pierdolnac...
+	 */
+	.text
+	.align		32
+	.globl		__flush_tlb_mm
+__flush_tlb_mm:		/* 19 insns */
+	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
+	ldxa		[%o1] ASI_DMMU, %g2
+	cmp		%g2, %o0
+	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
+	 mov		0x50, %g3
+	stxa		%g0, [%g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g3] ASI_IMMU_DEMAP
+	sethi		%hi(KERNBASE), %g3
+	flush		%g3
+	retl
+	 nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	.align		32
+	.globl		__flush_tlb_page
+__flush_tlb_page:	/* 22 insns */
+	/* %o0 = context, %o1 = vaddr */
+	rdpr		%pstate, %g7
+	andn		%g7, PSTATE_IE, %g2
+	wrpr		%g2, %pstate
+	mov		SECONDARY_CONTEXT, %o4
+	ldxa		[%o4] ASI_DMMU, %g2
+	stxa		%o0, [%o4] ASI_DMMU
+	andcc		%o1, 1, %g0
+	andn		%o1, 1, %o3
+	be,pn		%icc, 1f
+	 or		%o3, 0x10, %o3
+	stxa		%g0, [%o3] ASI_IMMU_DEMAP
+1:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
+	membar		#Sync
+	stxa		%g2, [%o4] ASI_DMMU
+	sethi		%hi(KERNBASE), %o4
+	flush		%o4
+	retl
+	 wrpr		%g7, 0x0, %pstate
+	nop
+	nop
+	nop
+	nop
+
+	.align		32
+	.globl		__flush_tlb_pending
+__flush_tlb_pending:	/* 27 insns */
+	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
+	rdpr		%pstate, %g7
+	sllx		%o1, 3, %o1
+	andn		%g7, PSTATE_IE, %g2
+	wrpr		%g2, %pstate
+	mov		SECONDARY_CONTEXT, %o4
+	ldxa		[%o4] ASI_DMMU, %g2
+	stxa		%o0, [%o4] ASI_DMMU
+1:	sub		%o1, (1 << 3), %o1
+	ldx		[%o2 + %o1], %o3
+	andcc		%o3, 1, %g0
+	andn		%o3, 1, %o3
+	be,pn		%icc, 2f
+	 or		%o3, 0x10, %o3
+	stxa		%g0, [%o3] ASI_IMMU_DEMAP
+2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%o1, 1b
+	 nop
+	stxa		%g2, [%o4] ASI_DMMU
+	sethi		%hi(KERNBASE), %o4
+	flush		%o4
+	retl
+	 wrpr		%g7, 0x0, %pstate
+	nop
+	nop
+	nop
+	nop
+
+	.align		32
+	.globl		__flush_tlb_kernel_range
+__flush_tlb_kernel_range:	/* 31 insns */
+	/* %o0=start, %o1=end */
+	cmp		%o0, %o1
+	be,pn		%xcc, 2f
+	 sub		%o1, %o0, %o3
+	srlx		%o3, 18, %o4
+	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
+	 sethi		%hi(PAGE_SIZE), %o4
+	sub		%o3, %o4, %o3
+	or		%o0, 0x20, %o0		! Nucleus
+1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
+	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%o3, 1b
+	 sub		%o3, %o4, %o3
+2:	sethi		%hi(KERNBASE), %o3
+	flush		%o3
+	retl
+	 nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+__spitfire_flush_tlb_kernel_range_slow:
+	mov		63 * 8, %o4
+1:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
+	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
+	bne,pn		%xcc, 2f
+	 mov		TLB_TAG_ACCESS, %o3
+	stxa		%g0, [%o3] ASI_IMMU
+	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
+	membar		#Sync
+2:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
+	andcc		%o3, 0x40, %g0
+	bne,pn		%xcc, 2f
+	 mov		TLB_TAG_ACCESS, %o3
+	stxa		%g0, [%o3] ASI_DMMU
+	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
+	membar		#Sync
+2:	sub		%o4, 8, %o4
+	brgez,pt	%o4, 1b
+	 nop
+	retl
+	 nop
+
+__spitfire_flush_tlb_mm_slow:
+	rdpr		%pstate, %g1
+	wrpr		%g1, PSTATE_IE, %pstate
+	stxa		%o0, [%o1] ASI_DMMU
+	stxa		%g0, [%g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g3] ASI_IMMU_DEMAP
+	flush		%g6
+	stxa		%g2, [%o1] ASI_DMMU
+	sethi		%hi(KERNBASE), %o1
+	flush		%o1
+	retl
+	 wrpr		%g1, 0, %pstate
+
+/*
+ * The following code flushes one page_size worth.
+ */
+	.section .kprobes.text, "ax"
+	.align		32
+	.globl		__flush_icache_page
+__flush_icache_page:	/* %o0 = phys_page */
+	srlx		%o0, PAGE_SHIFT, %o0
+	sethi		%hi(PAGE_OFFSET), %g1
+	sllx		%o0, PAGE_SHIFT, %o0
+	sethi		%hi(PAGE_SIZE), %g2
+	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
+	add		%o0, %g1, %o0
+1:	subcc		%g2, 32, %g2
+	bne,pt		%icc, 1b
+	 flush		%o0 + %g2
+	retl
+	 nop
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+
+#if (PAGE_SHIFT != 13)
+#error only page shift of 13 is supported by dcache flush
+#endif
+
+#define DTAG_MASK 0x3
+
+	/* This routine is Spitfire specific so the hardcoded
+	 * D-cache size and line-size are OK.
+	 */
+	.align		64
+	.globl		__flush_dcache_page
+__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
+	sethi		%hi(PAGE_OFFSET), %g1
+	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
+	sub		%o0, %g1, %o0			! physical address
+	srlx		%o0, 11, %o0			! make D-cache TAG
+	sethi		%hi(1 << 14), %o2		! D-cache size
+	sub		%o2, (1 << 5), %o2		! D-cache line size
+1:	ldxa		[%o2] ASI_DCACHE_TAG, %o3	! load D-cache TAG
+	andcc		%o3, DTAG_MASK, %g0		! Valid?
+	be,pn		%xcc, 2f			! Nope, branch
+	 andn		%o3, DTAG_MASK, %o3		! Clear valid bits
+	cmp		%o3, %o0			! TAG match?
+	bne,pt		%xcc, 2f			! Nope, branch
+	 nop
+	stxa		%g0, [%o2] ASI_DCACHE_TAG	! Invalidate TAG
+	membar		#Sync
+2:	brnz,pt		%o2, 1b
+	 sub		%o2, (1 << 5), %o2		! D-cache line size
+
+	/* The I-cache does not snoop local stores so we
+	 * better flush that too when necessary.
+	 */
+	brnz,pt		%o1, __flush_icache_page
+	 sllx		%o0, 11, %o0
+	retl
+	 nop
+
+#endif /* DCACHE_ALIASING_POSSIBLE */
+
+	.previous
+
+	/* Cheetah specific versions, patched at boot time. */
+__cheetah_flush_tlb_mm: /* 19 insns */
+	rdpr		%pstate, %g7
+	andn		%g7, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	mov		0x40, %g3
+	ldxa		[%o2] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
+	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
+	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
+	stxa		%o0, [%o2] ASI_DMMU
+	stxa		%g0, [%g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g3] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	sethi		%hi(KERNBASE), %o2
+	flush		%o2
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g7, 0x0, %pstate
+
+__cheetah_flush_tlb_page:	/* 22 insns */
+	/* %o0 = context, %o1 = vaddr */
+	rdpr		%pstate, %g7
+	andn		%g7, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o4
+	ldxa		[%o4] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
+	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
+	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
+	stxa		%o0, [%o4] ASI_DMMU
+	andcc		%o1, 1, %g0
+	be,pn		%icc, 1f
+	 andn		%o1, 1, %o3
+	stxa		%g0, [%o3] ASI_IMMU_DEMAP
+1:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
+	membar		#Sync
+	stxa		%g2, [%o4] ASI_DMMU
+	sethi		%hi(KERNBASE), %o4
+	flush		%o4
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g7, 0x0, %pstate
+
+__cheetah_flush_tlb_pending:	/* 27 insns */
+	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
+	rdpr		%pstate, %g7
+	sllx		%o1, 3, %o1
+	andn		%g7, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o4
+	ldxa		[%o4] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
+	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
+	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
+	stxa		%o0, [%o4] ASI_DMMU
+1:	sub		%o1, (1 << 3), %o1
+	ldx		[%o2 + %o1], %o3
+	andcc		%o3, 1, %g0
+	be,pn		%icc, 2f
+	 andn		%o3, 1, %o3
+	stxa		%g0, [%o3] ASI_IMMU_DEMAP
+2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
+	membar		#Sync
+	brnz,pt		%o1, 1b
+	 nop
+	stxa		%g2, [%o4] ASI_DMMU
+	sethi		%hi(KERNBASE), %o4
+	flush		%o4
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g7, 0x0, %pstate
+
+__cheetah_flush_tlb_kernel_range:	/* 31 insns */
+	/* %o0=start, %o1=end */
+	cmp		%o0, %o1
+	be,pn		%xcc, 2f
+	 sub		%o1, %o0, %o3
+	srlx		%o3, 18, %o4
+	brnz,pn		%o4, 3f
+	 sethi		%hi(PAGE_SIZE), %o4
+	sub		%o3, %o4, %o3
+	or		%o0, 0x20, %o0		! Nucleus
+1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
+	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%o3, 1b
+	 sub		%o3, %o4, %o3
+2:	sethi		%hi(KERNBASE), %o3
+	flush		%o3
+	retl
+	 nop
+3:	mov		0x80, %o4
+	stxa		%g0, [%o4] ASI_DMMU_DEMAP
+	membar		#Sync
+	stxa		%g0, [%o4] ASI_IMMU_DEMAP
+	membar		#Sync
+	retl
+	 nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+__cheetah_flush_dcache_page: /* 11 insns */
+	sethi		%hi(PAGE_OFFSET), %g1
+	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
+	sub		%o0, %g1, %o0
+	sethi		%hi(PAGE_SIZE), %o4
+1:	subcc		%o4, (1 << 5), %o4
+	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
+	membar		#Sync
+	bne,pt		%icc, 1b
+	 nop
+	retl		/* I-cache flush never needed on Cheetah, see callers. */
+	 nop
+#endif /* DCACHE_ALIASING_POSSIBLE */
+
+	/* Hypervisor specific versions, patched at boot time.  */
+__hypervisor_tlb_tl0_error:
+	save		%sp, -192, %sp
+	mov		%i0, %o0
+	call		hypervisor_tlbop_error
+	 mov		%i1, %o1
+	ret
+	 restore
+
+__hypervisor_flush_tlb_mm: /* 19 insns */
+	mov		%o0, %o2	/* ARG2: mmu context */
+	mov		0, %o0		/* ARG0: CPU lists unimplemented */
+	mov		0, %o1		/* ARG1: CPU lists unimplemented */
+	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
+	mov		HV_FAST_MMU_DEMAP_CTX, %o5
+	ta		HV_FAST_TRAP
+	brnz,pn		%o0, 1f
+	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
+	retl
+	 nop
+1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
+	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
+	 nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+__hypervisor_flush_tlb_page: /* 22 insns */
+	/* %o0 = context, %o1 = vaddr */
+	mov		%o0, %g2
+	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
+	mov		%g2, %o1	      /* ARG1: mmu context */
+	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
+	srlx		%o0, PAGE_SHIFT, %o0
+	sllx		%o0, PAGE_SHIFT, %o0
+	ta		HV_MMU_UNMAP_ADDR_TRAP
+	brnz,pn		%o0, 1f
+	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
+	retl
+	 nop
+1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
+	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
+	 nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+__hypervisor_flush_tlb_pending: /* 27 insns */
+	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
+	sllx		%o1, 3, %g1
+	mov		%o2, %g2
+	mov		%o0, %g3
+1:	sub		%g1, (1 << 3), %g1
+	ldx		[%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
+	mov		%g3, %o1	      /* ARG1: mmu context */
+	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
+	srlx		%o0, PAGE_SHIFT, %o0
+	sllx		%o0, PAGE_SHIFT, %o0
+	ta		HV_MMU_UNMAP_ADDR_TRAP
+	brnz,pn		%o0, 1f
+	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
+	brnz,pt		%g1, 1b
+	 nop
+	retl
+	 nop
+1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
+	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
+	 nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+__hypervisor_flush_tlb_kernel_range: /* 31 insns */
+	/* %o0=start, %o1=end */
+	cmp		%o0, %o1
+	be,pn		%xcc, 2f
+	 sub		%o1, %o0, %g2
+	srlx		%g2, 18, %g3
+	brnz,pn		%g3, 4f
+	 mov		%o0, %g1
+	sethi		%hi(PAGE_SIZE), %g3
+	sub		%g2, %g3, %g2
+1:	add		%g1, %g2, %o0	/* ARG0: virtual address */
+	mov		0, %o1		/* ARG1: mmu context */
+	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
+	ta		HV_MMU_UNMAP_ADDR_TRAP
+	brnz,pn		%o0, 3f
+	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
+	brnz,pt		%g2, 1b
+	 sub		%g2, %g3, %g2
+2:	retl
+	 nop
+3:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
+	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
+	 nop
+4:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
+	mov		0, %o1		/* ARG1: CPU lists unimplemented */
+	mov		0, %o2		/* ARG2: mmu context == nucleus */
+	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
+	mov		HV_FAST_MMU_DEMAP_CTX, %o5
+	ta		HV_FAST_TRAP
+	brnz,pn		%o0, 3b
+	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
+	retl
+	 nop
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+	/* XXX Niagara and friends have an 8K cache, so no aliasing is
+	 * XXX possible, but nothing explicit in the Hypervisor API
+	 * XXX guarantees this.
+	 */
+__hypervisor_flush_dcache_page:	/* 2 insns */
+	retl
+	 nop
+#endif
+
+tlb_patch_one:
+1:	lduw		[%o1], %g1
+	stw		%g1, [%o0]
+	flush		%o0
+	subcc		%o2, 1, %o2
+	add		%o1, 4, %o1
+	bne,pt		%icc, 1b
+	 add		%o0, 4, %o0
+	retl
+	 nop
+
+#ifdef CONFIG_SMP
+	/* These are all called by the slaves of a cross call, at
+	 * trap level 1, with interrupts fully disabled.
+	 *
+	 * Register usage:
+	 *   %g5	mm->context	(all tlb flushes)
+	 *   %g1	address arg 1	(tlb page and range flushes)
+	 *   %g7	address arg 2	(tlb range flush only)
+	 *
+	 *   %g6	scratch 1
+	 *   %g2	scratch 2
+	 *   %g3	scratch 3
+	 *   %g4	scratch 4
+	 */
+	.align		32
+	.globl		xcall_flush_tlb_mm
+xcall_flush_tlb_mm:	/* 24 insns */
+	mov		PRIMARY_CONTEXT, %g2
+	ldxa		[%g2] ASI_DMMU, %g3
+	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
+	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
+	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
+	stxa		%g5, [%g2] ASI_DMMU
+	mov		0x40, %g4
+	stxa		%g0, [%g4] ASI_DMMU_DEMAP
+	stxa		%g0, [%g4] ASI_IMMU_DEMAP
+	stxa		%g3, [%g2] ASI_DMMU
+	retry
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	.globl		xcall_flush_tlb_page
+xcall_flush_tlb_page:	/* 20 insns */
+	/* %g5=context, %g1=vaddr */
+	mov		PRIMARY_CONTEXT, %g4
+	ldxa		[%g4] ASI_DMMU, %g2
+	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
+	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
+	or		%g5, %g4, %g5
+	mov		PRIMARY_CONTEXT, %g4
+	stxa		%g5, [%g4] ASI_DMMU
+	andcc		%g1, 0x1, %g0
+	be,pn		%icc, 2f
+	 andn		%g1, 0x1, %g5
+	stxa		%g0, [%g5] ASI_IMMU_DEMAP
+2:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
+	membar		#Sync
+	stxa		%g2, [%g4] ASI_DMMU
+	retry
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	.globl		xcall_flush_tlb_kernel_range
+xcall_flush_tlb_kernel_range:	/* 44 insns */
+	sethi		%hi(PAGE_SIZE - 1), %g2
+	or		%g2, %lo(PAGE_SIZE - 1), %g2
+	andn		%g1, %g2, %g1
+	andn		%g7, %g2, %g7
+	sub		%g7, %g1, %g3
+	srlx		%g3, 18, %g2
+	brnz,pn		%g2, 2f
+	 sethi		%hi(PAGE_SIZE), %g2
+	sub		%g3, %g2, %g3
+	or		%g1, 0x20, %g1		! Nucleus
+1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%g3, 1b
+	 sub		%g3, %g2, %g3
+	retry
+2:	mov		63 * 8, %g1
+1:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
+	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
+	bne,pn		%xcc, 2f
+	 mov		TLB_TAG_ACCESS, %g2
+	stxa		%g0, [%g2] ASI_IMMU
+	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
+	membar		#Sync
+2:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
+	andcc		%g2, 0x40, %g0
+	bne,pn		%xcc, 2f
+	 mov		TLB_TAG_ACCESS, %g2
+	stxa		%g0, [%g2] ASI_DMMU
+	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
+	membar		#Sync
+2:	sub		%g1, 8, %g1
+	brgez,pt	%g1, 1b
+	 nop
+	retry
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+	/* This runs in a very controlled environment, so we do
+	 * not need to worry about BH races etc.
+	 */
+	.globl		xcall_sync_tick
+xcall_sync_tick:
+
+661:	rdpr		%pstate, %g2
+	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
+	.section	.sun4v_2insn_patch, "ax"
+	.word		661b
+	nop
+	nop
+	.previous
+
+	rdpr		%pil, %g2
+	wrpr		%g0, PIL_NORMAL_MAX, %pil
+	sethi		%hi(109f), %g7
+	b,pt		%xcc, etrap_irq
+109:	 or		%g7, %lo(109b), %g7
+#ifdef CONFIG_TRACE_IRQFLAGS
+	call		trace_hardirqs_off
+	 nop
+#endif
+	call		smp_synchronize_tick_client
+	 nop
+	b		rtrap_xcall
+	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
+
+	.globl		xcall_fetch_glob_regs
+xcall_fetch_glob_regs:
+	sethi		%hi(global_cpu_snapshot), %g1
+	or		%g1, %lo(global_cpu_snapshot), %g1
+	__GET_CPUID(%g2)
+	sllx		%g2, 6, %g3
+	add		%g1, %g3, %g1
+	rdpr		%tstate, %g7
+	stx		%g7, [%g1 + GR_SNAP_TSTATE]
+	rdpr		%tpc, %g7
+	stx		%g7, [%g1 + GR_SNAP_TPC]
+	rdpr		%tnpc, %g7
+	stx		%g7, [%g1 + GR_SNAP_TNPC]
+	stx		%o7, [%g1 + GR_SNAP_O7]
+	stx		%i7, [%g1 + GR_SNAP_I7]
+	/* Don't try this at home kids... */
+	rdpr		%cwp, %g3
+	sub		%g3, 1, %g7
+	wrpr		%g7, %cwp
+	mov		%i7, %g7
+	wrpr		%g3, %cwp
+	stx		%g7, [%g1 + GR_SNAP_RPC]
+	sethi		%hi(trap_block), %g7
+	or		%g7, %lo(trap_block), %g7
+	sllx		%g2, TRAP_BLOCK_SZ_SHIFT, %g2
+	add		%g7, %g2, %g7
+	ldx		[%g7 + TRAP_PER_CPU_THREAD], %g3
+	stx		%g3, [%g1 + GR_SNAP_THREAD]
+	retry
+
+	.globl		xcall_fetch_glob_pmu
+xcall_fetch_glob_pmu:
+	sethi		%hi(global_cpu_snapshot), %g1
+	or		%g1, %lo(global_cpu_snapshot), %g1
+	__GET_CPUID(%g2)
+	sllx		%g2, 6, %g3
+	add		%g1, %g3, %g1
+	rd		%pic, %g7
+	stx		%g7, [%g1 + (4 * 8)]
+	rd		%pcr, %g7
+	stx		%g7, [%g1 + (0 * 8)]
+	retry
+
+	.globl		xcall_fetch_glob_pmu_n4
+xcall_fetch_glob_pmu_n4:
+	sethi		%hi(global_cpu_snapshot), %g1
+	or		%g1, %lo(global_cpu_snapshot), %g1
+	__GET_CPUID(%g2)
+	sllx		%g2, 6, %g3
+	add		%g1, %g3, %g1
+
+	ldxa		[%g0] ASI_PIC, %g7
+	stx		%g7, [%g1 + (4 * 8)]
+	mov		0x08, %g3
+	ldxa		[%g3] ASI_PIC, %g7
+	stx		%g7, [%g1 + (5 * 8)]
+	mov		0x10, %g3
+	ldxa		[%g3] ASI_PIC, %g7
+	stx		%g7, [%g1 + (6 * 8)]
+	mov		0x18, %g3
+	ldxa		[%g3] ASI_PIC, %g7
+	stx		%g7, [%g1 + (7 * 8)]
+
+	mov		%o0, %g2
+	mov		%o1, %g3
+	mov		%o5, %g7
+
+	mov		HV_FAST_VT_GET_PERFREG, %o5
+	mov		3, %o0
+	ta		HV_FAST_TRAP
+	stx		%o1, [%g1 + (3 * 8)]
+	mov		HV_FAST_VT_GET_PERFREG, %o5
+	mov		2, %o0
+	ta		HV_FAST_TRAP
+	stx		%o1, [%g1 + (2 * 8)]
+	mov		HV_FAST_VT_GET_PERFREG, %o5
+	mov		1, %o0
+	ta		HV_FAST_TRAP
+	stx		%o1, [%g1 + (1 * 8)]
+	mov		HV_FAST_VT_GET_PERFREG, %o5
+	mov		0, %o0
+	ta		HV_FAST_TRAP
+	stx		%o1, [%g1 + (0 * 8)]
+
+	mov		%g2, %o0
+	mov		%g3, %o1
+	mov		%g7, %o5
+
+	retry
+
+__cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
+	sethi		%hi(PAGE_SIZE - 1), %g2
+	or		%g2, %lo(PAGE_SIZE - 1), %g2
+	andn		%g1, %g2, %g1
+	andn		%g7, %g2, %g7
+	sub		%g7, %g1, %g3
+	srlx		%g3, 18, %g2
+	brnz,pn		%g2, 2f
+	 sethi		%hi(PAGE_SIZE), %g2
+	sub		%g3, %g2, %g3
+	or		%g1, 0x20, %g1		! Nucleus
+1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%g3, 1b
+	 sub		%g3, %g2, %g3
+	retry
+2:	mov		0x80, %g2
+	stxa		%g0, [%g2] ASI_DMMU_DEMAP
+	membar		#Sync
+	stxa		%g0, [%g2] ASI_IMMU_DEMAP
+	membar		#Sync
+	retry
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+	.align		32
+	.globl		xcall_flush_dcache_page_cheetah
+xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
+	sethi		%hi(PAGE_SIZE), %g3
+1:	subcc		%g3, (1 << 5), %g3
+	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
+	membar		#Sync
+	bne,pt		%icc, 1b
+	 nop
+	retry
+	nop
+#endif /* DCACHE_ALIASING_POSSIBLE */
+
+	.globl		xcall_flush_dcache_page_spitfire
+xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
+				     %g7 == kernel page virtual address
+				     %g5 == (page->mapping != NULL)  */
+#ifdef DCACHE_ALIASING_POSSIBLE
+	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
+	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
+	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
+1:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
+	andcc		%g2, 0x3, %g0
+	be,pn		%xcc, 2f
+	 andn		%g2, 0x3, %g2
+	cmp		%g2, %g1
+
+	bne,pt		%xcc, 2f
+	 nop
+	stxa		%g0, [%g3] ASI_DCACHE_TAG
+	membar		#Sync
+2:	cmp		%g3, 0
+	bne,pt		%xcc, 1b
+	 sub		%g3, (1 << 5), %g3
+
+	brz,pn		%g5, 2f
+#endif /* DCACHE_ALIASING_POSSIBLE */
+	 sethi		%hi(PAGE_SIZE), %g3
+
+1:	flush		%g7
+	subcc		%g3, (1 << 5), %g3
+	bne,pt		%icc, 1b
+	 add		%g7, (1 << 5), %g7
+
+2:	retry
+	nop
+	nop
+
+	/* %g5:	error
+	 * %g6:	tlb op
+	 */
+__hypervisor_tlb_xcall_error:
+	mov	%g5, %g4
+	mov	%g6, %g5
+	ba,pt	%xcc, etrap
+	 rd	%pc, %g7
+	mov	%l4, %o0
+	call	hypervisor_tlbop_error_xcall
+	 mov	%l5, %o1
+	ba,a,pt	%xcc, rtrap
+
+	.globl		__hypervisor_xcall_flush_tlb_mm
+__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
+	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
+	mov		%o0, %g2
+	mov		%o1, %g3
+	mov		%o2, %g4
+	mov		%o3, %g1
+	mov		%o5, %g7
+	clr		%o0		/* ARG0: CPU lists unimplemented */
+	clr		%o1		/* ARG1: CPU lists unimplemented */
+	mov		%g5, %o2	/* ARG2: mmu context */
+	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
+	mov		HV_FAST_MMU_DEMAP_CTX, %o5
+	ta		HV_FAST_TRAP
+	mov		HV_FAST_MMU_DEMAP_CTX, %g6
+	brnz,pn		%o0, 1f
+	 mov		%o0, %g5
+	mov		%g2, %o0
+	mov		%g3, %o1
+	mov		%g4, %o2
+	mov		%g1, %o3
+	mov		%g7, %o5
+	membar		#Sync
+	retry
+1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
+	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
+	 nop
+
+	.globl		__hypervisor_xcall_flush_tlb_page
+__hypervisor_xcall_flush_tlb_page: /* 20 insns */
+	/* %g5=ctx, %g1=vaddr */
+	mov		%o0, %g2
+	mov		%o1, %g3
+	mov		%o2, %g4
+	mov		%g1, %o0	        /* ARG0: virtual address */
+	mov		%g5, %o1		/* ARG1: mmu context */
+	mov		HV_MMU_ALL, %o2		/* ARG2: flags */
+	srlx		%o0, PAGE_SHIFT, %o0
+	sllx		%o0, PAGE_SHIFT, %o0
+	ta		HV_MMU_UNMAP_ADDR_TRAP
+	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
+	brnz,a,pn	%o0, 1f
+	 mov		%o0, %g5
+	mov		%g2, %o0
+	mov		%g3, %o1
+	mov		%g4, %o2
+	membar		#Sync
+	retry
+1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
+	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
+	 nop
+
+	.globl		__hypervisor_xcall_flush_tlb_kernel_range
+__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
+	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
+	sethi		%hi(PAGE_SIZE - 1), %g2
+	or		%g2, %lo(PAGE_SIZE - 1), %g2
+	andn		%g1, %g2, %g1
+	andn		%g7, %g2, %g7
+	sub		%g7, %g1, %g3
+	srlx		%g3, 18, %g7
+	add		%g2, 1, %g2
+	sub		%g3, %g2, %g3
+	mov		%o0, %g2
+	mov		%o1, %g4
+	brnz,pn		%g7, 2f
+	 mov		%o2, %g7
+1:	add		%g1, %g3, %o0	/* ARG0: virtual address */
+	mov		0, %o1		/* ARG1: mmu context */
+	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
+	ta		HV_MMU_UNMAP_ADDR_TRAP
+	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
+	brnz,pn		%o0, 1f
+	 mov		%o0, %g5
+	sethi		%hi(PAGE_SIZE), %o2
+	brnz,pt		%g3, 1b
+	 sub		%g3, %o2, %g3
+5:	mov		%g2, %o0
+	mov		%g4, %o1
+	mov		%g7, %o2
+	membar		#Sync
+	retry
+1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
+	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
+	 nop
+2:	mov		%o3, %g1
+	mov		%o5, %g3
+	mov		0, %o0		/* ARG0: CPU lists unimplemented */
+	mov		0, %o1		/* ARG1: CPU lists unimplemented */
+	mov		0, %o2		/* ARG2: mmu context == nucleus */
+	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
+	mov		HV_FAST_MMU_DEMAP_CTX, %o5
+	ta		HV_FAST_TRAP
+	mov		%g1, %o3
+	brz,pt		%o0, 5b
+	 mov		%g3, %o5
+	mov		HV_FAST_MMU_DEMAP_CTX, %g6
+	ba,pt		%xcc, 1b
+	 clr		%g5
+
+	/* These just get rescheduled to PIL vectors. */
+	.globl		xcall_call_function
+xcall_call_function:
+	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
+	retry
+
+	.globl		xcall_call_function_single
+xcall_call_function_single:
+	wr		%g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
+	retry
+
+	.globl		xcall_receive_signal
+xcall_receive_signal:
+	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
+	retry
+
+	.globl		xcall_capture
+xcall_capture:
+	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
+	retry
+
+#ifdef CONFIG_KGDB
+	.globl		xcall_kgdb_capture
+xcall_kgdb_capture:
+	wr		%g0, (1 << PIL_KGDB_CAPTURE), %set_softint
+	retry
+#endif
+
+#endif /* CONFIG_SMP */
+
+	.globl		cheetah_patch_cachetlbops
+cheetah_patch_cachetlbops:
+	save		%sp, -128, %sp
+
+	sethi		%hi(__flush_tlb_mm), %o0
+	or		%o0, %lo(__flush_tlb_mm), %o0
+	sethi		%hi(__cheetah_flush_tlb_mm), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
+	call		tlb_patch_one
+	 mov		19, %o2
+
+	sethi		%hi(__flush_tlb_page), %o0
+	or		%o0, %lo(__flush_tlb_page), %o0
+	sethi		%hi(__cheetah_flush_tlb_page), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
+	call		tlb_patch_one
+	 mov		22, %o2
+
+	sethi		%hi(__flush_tlb_pending), %o0
+	or		%o0, %lo(__flush_tlb_pending), %o0
+	sethi		%hi(__cheetah_flush_tlb_pending), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
+	call		tlb_patch_one
+	 mov		27, %o2
+
+	sethi		%hi(__flush_tlb_kernel_range), %o0
+	or		%o0, %lo(__flush_tlb_kernel_range), %o0
+	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
+	call		tlb_patch_one
+	 mov		31, %o2
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+	sethi		%hi(__flush_dcache_page), %o0
+	or		%o0, %lo(__flush_dcache_page), %o0
+	sethi		%hi(__cheetah_flush_dcache_page), %o1
+	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
+	call		tlb_patch_one
+	 mov		11, %o2
+#endif /* DCACHE_ALIASING_POSSIBLE */
+
+#ifdef CONFIG_SMP
+	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
+	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
+	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
+	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
+	call		tlb_patch_one
+	 mov		44, %o2
+#endif /* CONFIG_SMP */
+
+	ret
+	 restore
+
+	.globl		hypervisor_patch_cachetlbops
+hypervisor_patch_cachetlbops:
+	save		%sp, -128, %sp
+
+	sethi		%hi(__flush_tlb_mm), %o0
+	or		%o0, %lo(__flush_tlb_mm), %o0
+	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
+	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
+	call		tlb_patch_one
+	 mov		19, %o2
+
+	sethi		%hi(__flush_tlb_page), %o0
+	or		%o0, %lo(__flush_tlb_page), %o0
+	sethi		%hi(__hypervisor_flush_tlb_page), %o1
+	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
+	call		tlb_patch_one
+	 mov		22, %o2
+
+	sethi		%hi(__flush_tlb_pending), %o0
+	or		%o0, %lo(__flush_tlb_pending), %o0
+	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
+	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
+	call		tlb_patch_one
+	 mov		27, %o2
+
+	sethi		%hi(__flush_tlb_kernel_range), %o0
+	or		%o0, %lo(__flush_tlb_kernel_range), %o0
+	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
+	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
+	call		tlb_patch_one
+	 mov		31, %o2
+
+#ifdef DCACHE_ALIASING_POSSIBLE
+	sethi		%hi(__flush_dcache_page), %o0
+	or		%o0, %lo(__flush_dcache_page), %o0
+	sethi		%hi(__hypervisor_flush_dcache_page), %o1
+	or		%o1, %lo(__hypervisor_flush_dcache_page), %o1
+	call		tlb_patch_one
+	 mov		2, %o2
+#endif /* DCACHE_ALIASING_POSSIBLE */
+
+#ifdef CONFIG_SMP
+	sethi		%hi(xcall_flush_tlb_mm), %o0
+	or		%o0, %lo(xcall_flush_tlb_mm), %o0
+	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
+	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
+	call		tlb_patch_one
+	 mov		24, %o2
+
+	sethi		%hi(xcall_flush_tlb_page), %o0
+	or		%o0, %lo(xcall_flush_tlb_page), %o0
+	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
+	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
+	call		tlb_patch_one
+	 mov		20, %o2
+
+	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
+	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
+	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
+	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
+	call		tlb_patch_one
+	 mov		44, %o2
+#endif /* CONFIG_SMP */
+
+	ret
+	 restore
diff --git a/arch/sparc/mm/viking.S b/arch/sparc/mm/viking.S
new file mode 100644
index 000000000..48f062de7
--- /dev/null
+++ b/arch/sparc/mm/viking.S
@@ -0,0 +1,284 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * viking.S: High speed Viking cache/mmu operations
+ *
+ * Copyright (C) 1997  Eddie C. Dost  (ecd@skynet.be)
+ * Copyright (C) 1997,1998,1999  Jakub Jelinek  (jj@ultra.linux.cz)
+ * Copyright (C) 1999  Pavel Semerad  (semerad@ss1000.ms.mff.cuni.cz)
+ */
+
+#include <asm/ptrace.h>
+#include <asm/psr.h>
+#include <asm/asm-offsets.h>
+#include <asm/asi.h>
+#include <asm/mxcc.h>
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <asm/pgtsrmmu.h>
+#include <asm/viking.h>
+
+#ifdef CONFIG_SMP
+	.data
+	.align	4
+sun4dsmp_flush_tlb_spin:
+	.word	0
+#endif
+
+	.text
+	.align	4
+
+	.globl	viking_flush_cache_all, viking_flush_cache_mm
+	.globl	viking_flush_cache_range, viking_flush_cache_page
+	.globl	viking_flush_page, viking_mxcc_flush_page
+	.globl	viking_flush_page_for_dma, viking_flush_page_to_ram
+	.globl	viking_flush_sig_insns
+	.globl	viking_flush_tlb_all, viking_flush_tlb_mm
+	.globl	viking_flush_tlb_range, viking_flush_tlb_page
+
+viking_flush_page:
+	sethi	%hi(PAGE_OFFSET), %g2
+	sub	%o0, %g2, %g3
+	srl	%g3, 12, %g1		! ppage >> 12
+
+	clr	%o1			! set counter, 0 - 127
+	sethi	%hi(PAGE_OFFSET + PAGE_SIZE - 0x80000000), %o3
+	sethi	%hi(0x80000000), %o4
+	sethi	%hi(VIKING_PTAG_VALID), %o5
+	sethi	%hi(2*PAGE_SIZE), %o0
+	sethi	%hi(PAGE_SIZE), %g7
+	clr	%o2			! block counter, 0 - 3
+5:
+	sll	%o1, 5, %g4
+	or	%g4, %o4, %g4		! 0x80000000 | (set << 5)
+
+	sll	%o2, 26, %g5		! block << 26
+6:
+	or	%g5, %g4, %g5
+	ldda	[%g5] ASI_M_DATAC_TAG, %g2
+	cmp	%g3, %g1		! ptag == ppage?
+	bne	7f
+	 inc	%o2
+
+	andcc	%g2, %o5, %g0		! ptag VALID?
+	be	7f
+	 add	%g4, %o3, %g2		! (PAGE_OFFSET + PAGE_SIZE) | (set << 5)
+	ld	[%g2], %g3
+	ld	[%g2 + %g7], %g3
+	add	%g2, %o0, %g2
+	ld	[%g2], %g3
+	ld	[%g2 + %g7], %g3
+	add	%g2, %o0, %g2
+	ld	[%g2], %g3
+	ld	[%g2 + %g7], %g3
+	add	%g2, %o0, %g2
+	ld	[%g2], %g3
+	b	8f
+	 ld	[%g2 + %g7], %g3
+
+7:
+	cmp	%o2, 3
+	ble	6b
+	 sll	%o2, 26, %g5			! block << 26
+
+8:	inc	%o1
+	cmp	%o1, 0x7f
+	ble	5b
+	 clr	%o2
+
+9:	retl
+	 nop
+
+viking_mxcc_flush_page:
+	sethi	%hi(PAGE_OFFSET), %g2
+	sub	%o0, %g2, %g3
+	sub	%g3, -PAGE_SIZE, %g3		! ppage + PAGE_SIZE
+	sethi	%hi(MXCC_SRCSTREAM), %o3	! assume %hi(MXCC_SRCSTREAM) == %hi(MXCC_DESTSTREAM)
+	mov	0x10, %g2			! set cacheable bit
+	or	%o3, %lo(MXCC_SRCSTREAM), %o2
+	or	%o3, %lo(MXCC_DESSTREAM), %o3
+	sub	%g3, MXCC_STREAM_SIZE, %g3
+6:
+	stda	%g2, [%o2] ASI_M_MXCC
+	stda	%g2, [%o3] ASI_M_MXCC
+	andncc	%g3, PAGE_MASK, %g0
+	bne	6b
+	 sub	%g3, MXCC_STREAM_SIZE, %g3
+
+9:	retl
+	 nop
+
+viking_flush_cache_page:
+viking_flush_cache_range:
+#ifndef CONFIG_SMP
+	ld	[%o0 + VMA_VM_MM], %o0
+#endif
+viking_flush_cache_mm:
+#ifndef CONFIG_SMP
+	ld	[%o0 + AOFF_mm_context], %g1
+	cmp	%g1, -1
+	bne	viking_flush_cache_all
+	 nop
+	b,a	viking_flush_cache_out
+#endif
+viking_flush_cache_all:
+	WINDOW_FLUSH(%g4, %g5)
+viking_flush_cache_out:
+	retl
+	 nop
+
+viking_flush_tlb_all:
+	mov	0x400, %g1
+	retl
+	 sta	%g0, [%g1] ASI_M_FLUSH_PROBE
+
+viking_flush_tlb_mm:
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o1
+	lda	[%g1] ASI_M_MMUREGS, %g5
+#ifndef CONFIG_SMP
+	cmp	%o1, -1
+	be	1f
+#endif
+	mov	0x300, %g2
+	sta	%o1, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+#ifndef CONFIG_SMP
+1:	retl
+	 nop
+#endif
+
+viking_flush_tlb_range:
+	ld	[%o0 + VMA_VM_MM], %o0
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o3
+	lda	[%g1] ASI_M_MMUREGS, %g5
+#ifndef CONFIG_SMP
+	cmp	%o3, -1
+	be	2f
+#endif
+	sethi	%hi(~((1 << PGDIR_SHIFT) - 1)), %o4
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	and	%o1, %o4, %o1
+	add	%o1, 0x200, %o1
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+1:	sub	%o1, %o4, %o1
+	cmp	%o1, %o2
+	blu,a	1b
+	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+#ifndef CONFIG_SMP
+2:	retl
+	 nop
+#endif
+
+viking_flush_tlb_page:
+	ld	[%o0 + VMA_VM_MM], %o0
+	mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o3
+	lda	[%g1] ASI_M_MMUREGS, %g5
+#ifndef CONFIG_SMP
+	cmp	%o3, -1
+	be	1f
+#endif
+	and	%o1, PAGE_MASK, %o1
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	retl
+	 sta	%g5, [%g1] ASI_M_MMUREGS
+#ifndef CONFIG_SMP
+1:	retl
+	 nop
+#endif
+
+viking_flush_page_to_ram:
+viking_flush_page_for_dma:
+viking_flush_sig_insns:
+	retl
+	 nop
+
+#ifdef CONFIG_SMP
+	.globl	sun4dsmp_flush_tlb_all, sun4dsmp_flush_tlb_mm
+	.globl	sun4dsmp_flush_tlb_range, sun4dsmp_flush_tlb_page
+sun4dsmp_flush_tlb_all:
+	sethi	%hi(sun4dsmp_flush_tlb_spin), %g3
+1:	ldstub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	tst	%g5
+	bne	2f
+	 mov	0x400, %g1
+	sta	%g0, [%g1] ASI_M_FLUSH_PROBE
+	retl
+	 stb	%g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)]
+2:	tst	%g5
+	bne,a	2b
+	 ldub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	b,a	1b
+
+sun4dsmp_flush_tlb_mm:
+	sethi	%hi(sun4dsmp_flush_tlb_spin), %g3
+1:	ldstub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	tst	%g5
+	bne	2f
+	 mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + AOFF_mm_context], %o1
+	lda	[%g1] ASI_M_MMUREGS, %g5
+	mov	0x300, %g2
+	sta	%o1, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
+	sta	%g5, [%g1] ASI_M_MMUREGS
+	retl
+	 stb	%g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)]
+2:	tst	%g5
+	bne,a	2b
+	 ldub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	b,a	1b
+
+sun4dsmp_flush_tlb_range:
+	sethi	%hi(sun4dsmp_flush_tlb_spin), %g3
+1:	ldstub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	tst	%g5
+	bne	3f
+	 mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + VMA_VM_MM], %o0
+	ld	[%o0 + AOFF_mm_context], %o3
+	lda	[%g1] ASI_M_MMUREGS, %g5
+	sethi	%hi(~((1 << PGDIR_SHIFT) - 1)), %o4
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	and	%o1, %o4, %o1
+	add	%o1, 0x200, %o1
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+2:	sub	%o1, %o4, %o1
+	cmp	%o1, %o2
+	blu,a	2b
+	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	sta	%g5, [%g1] ASI_M_MMUREGS
+	retl
+	 stb	%g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)]
+3:	tst	%g5
+	bne,a	3b
+	 ldub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	b,a	1b
+
+sun4dsmp_flush_tlb_page:
+	sethi	%hi(sun4dsmp_flush_tlb_spin), %g3
+1:	ldstub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	tst	%g5
+	bne	2f
+	 mov	SRMMU_CTX_REG, %g1
+	ld	[%o0 + VMA_VM_MM], %o0
+	ld	[%o0 + AOFF_mm_context], %o3
+	lda	[%g1] ASI_M_MMUREGS, %g5
+	and	%o1, PAGE_MASK, %o1
+	sta	%o3, [%g1] ASI_M_MMUREGS
+	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
+	sta	%g5, [%g1] ASI_M_MMUREGS
+	retl
+	 stb	%g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)]
+2:	tst	%g5
+	bne,a	2b
+	 ldub	[%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5
+	b,a	1b
+	 nop
+#endif